:W&Bá¾îéÕWM bx3tre1aª; 9 /0.29.0+7872fd9d9f86ca5c50617e19cb168122b7f959480.70.0<’ ù‚" er0l7f72du0w bx3tre1aŠÕ bx3tre1ajuju11-uc-san-diegoSPAtop"™ _wandb‚{}  batch_size‚256  dataset_limit‚0.2  dropout‚0.07143685137493079 epochs‚10  hidden_dim‚128 ' learning_rate‚0.0001266856927550707  num_attention_heads‚8  num_branch_encoder_layers‚3  num_embedding_layers‚6  num_encoder_layers‚6Bjjtttt_vanilla_bx3tre1ab88oquqqwj)jj-spatop-sweepagent-tttt-vanilla-1-k2mn8Š ÔæÔ¨•œ š3 (H¨¸hp"3.12.10*0.29.0j linux-x86_64u¸"©7 bx3tre1a&  _runtime‚0  _wandb.runtime‚0N®HpiÒã -Linux-6.8.0-136-generic-x86_64-with-glibc2.36CPython 3.12.10 ÓæÔ°‡ÇG*-of*=/data/spatop/options_files/sweep_tttt_jj/config_bx3tre1a.json*-l* /scratch/logs*-n*jjtttt_vanilla_bx3tre1a2-m spanet.trainRjacksonjulian2014@gmail.comZ /scratch/logsb)jj-spatop-sweepagent-tttt-vanilla-1-k2mn8r/opt/conda/bin/pythonº sy40sq0ma74hzb1ahf2v7h09uqs3trimH@ðSÉÒÀ€ˆ€’NVIDIA A100-SXM4-80GB˜¢ /€€€¥È.€ÀøÊ  ª€àãÓ½ÂS NVIDIA A100-SXM4-80GB€€€€À€6"Ampere*(GPU-78f24734-e3a1-59a7-ce1b-bb4283f77c97Ê13.2º sy40sq0ma74hzb1ahf2v7h09uqs3trimµª½Ç2  requirements.txt'»Š÷W bx3tre1aZF (H¨¸ (H¨¸ (hpð"3.12.10*0.29.0j linux-x86_64g1^6Y bx3tre1aZH (H¨¸ (H¨¸ (8hpð"3.12.10*0.29.0j linux-x86_64p!EØ*  bx3tre1ab trainer/global_step2(K"„\  bx3tre1aZK (H¨¸ (H¨¸(8hpð"3.12.10*0.29.0j linux-x86_64VÖ*l/  bx3tre1ab*"trainer/global_step2›+Ébb ?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak"trainer/global_step2:JP&»„d?b` =DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak"trainer/global_step2:JP•ž¯¾g@bc @DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed"trainer/global_step2:JP¦ïýpAbl IDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current"trainer/global_step2:JP0àKÇxBbt QDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated"trainer/global_step2:JP9}–jCbf CDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated"trainer/global_step2:JPXÙ—ÙoDbk HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed"trainer/global_step2:JPb¶"×eEba >DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak"trainer/global_step2:JPœ ¥t`Fb\ 9DeviceStatsMonitor.on_train_batch_start/segment.all.freed"trainer/global_step2:JPAºÚˆkGbg DDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated"trainer/global_step2:JPx¬¸ÛlHbh EDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak"trainer/global_step2:JPà  QWIbS 0DeviceStatsMonitor.on_train_batch_start/num_ooms"trainer/global_step2:JPÒ>µËhJbd ADeviceStatsMonitor.on_train_batch_start/oversize_segments.current"trainer/global_step2:JPÅðÚÕsKbo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated"trainer/global_step2:JP­U/'nLbj GDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current"trainer/global_step2:JP“HmMbi FDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak"trainer/global_step2:JP¤4“iNbe BDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current"trainer/global_step2:JP]¬VooObk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed"trainer/global_step2:JPS¥l–gPbc @DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated"trainer/global_step2:JPÏçLGsQbo LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated"trainer/global_step2:JP ¬oRbk HDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current"trainer/global_step2:JPíðqSbm JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated"trainer/global_step2:JPÇk‡vgTbc @DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak"trainer/global_step2:JPkt”®sUbo LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak"trainer/global_step2:JPœ{Û™gVbc @DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed"trainer/global_step2:JP§ GzfWbb ?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed"trainer/global_step2:JP¸ëTVnXbj GDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated"trainer/global_step2:JPÏcqYbm JDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current"trainer/global_step2:JPÄiÖInZbj GDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak"trainer/global_step2:JP´Ñ½__[b[ 8DeviceStatsMonitor.on_train_batch_start/active.all.freed"trainer/global_step2:JPìYzHl\bh EDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed"trainer/global_step2:JPçXÃl]bh EDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current"trainer/global_step2:JPÕoyAp^bl IDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current"trainer/global_step2:JPS¨; n_bj GDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current"trainer/global_step2:JPª'¹c`b_ DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak"trainer/global_step2:JPâ‡ú®ebba >DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed"trainer/global_step2:JPÄ qcbm JDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current"trainer/global_step2:JPËn‹ndbj GDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak"trainer/global_step2:JPGú_4tebp MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed"trainer/global_step2:JP;Lpjfbf CDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current"trainer/global_step2:JP"È·jgbf CDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed"trainer/global_step2:JPˆ>•¥mhbi FDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak"trainer/global_step2:JP^É"^ibZ 7DeviceStatsMonitor.on_train_batch_start/num_device_free"trainer/global_step2:JP­)H{fjbb ?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak"trainer/global_step2:JP¤ ‚kkbg DDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated"trainer/global_step2:JPŸÑflbb ?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak"trainer/global_step2:JPJèlmbh EDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed"trainer/global_step2:JP+`“Binbe BDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current"trainer/global_step2:JP~1Íiobe BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated"trainer/global_step2:JP§T Ajpbf CDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current"trainer/global_step2:JP—‡4dnqbj GDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed"trainer/global_step2:JP=dr vrbr ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current"trainer/global_step2:JPÑ~ssbo LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated"trainer/global_step2:JPíþ-y]tbY 6DeviceStatsMonitor.on_train_batch_start/max_split_size"trainer/global_step2:JP· =rubn KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated"trainer/global_step2:JP….´´ivbe BDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current"trainer/global_step2:JPÆC' cwb_ DeviceStatsMonitor.on_train_batch_start/allocation.all.current"trainer/global_step2:JPn3r~bn KDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated"trainer/global_step2:JP€h,Îpbl IDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current"trainer/global_step2:JPÉák€bf CDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed"trainer/global_step2:JPÞ÷nbi FDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak"trainer/global_step2:JPWX°So‚bj GDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed"trainer/global_step2:JP•4|U`ƒb[ 8DeviceStatsMonitor.on_train_batch_start/segment.all.peak"trainer/global_step2:JP?PMxf„ba >DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak"trainer/global_step2:JPRûi?s…bn KDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated"trainer/global_step2:JP£Á:Wp†bk HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed"trainer/global_step2:JP5*ÛKi‡bd ADeviceStatsMonitor.on_train_batch_start/active.large_pool.current"trainer/global_step2:JPºRshgˆbb ?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak"trainer/global_step2:JPdyÙbj‰be BDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed"trainer/global_step2:JPRj[ÉqŠbl IDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current"trainer/global_step2:JPn»Û;m‹bh EDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated"trainer/global_step2:JP¢Å£åjŒbe BDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak"trainer/global_step2:JP¯Â[nbi FDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated"trainer/global_step2:JPÏ"­hŽbc @DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current"trainer/global_step2:JPA„ælbg DDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak"trainer/global_step2:JPîvnémbh EDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current"trainer/global_step2:JP¼ùl’j‘be BDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current"trainer/global_step2:JPP¹–`’b[ 8DeviceStatsMonitor.on_train_batch_start/num_device_alloc"trainer/global_step2:JPÅO´Fo“bj GDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed"trainer/global_step2:JP5\aq”bl IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated"trainer/global_step2:JPñ•‡åp•bk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed"trainer/global_step2:JP••ÕÉy–bt QDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated"trainer/global_step2:JP’¬]ñw—br ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current"trainer/global_step2:JPø6B¥d˜b_ DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚645582 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 332015104 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 41237680866 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10192158720 trainer/global_step‚49 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 40735264768 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1648136 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 41390860288 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 3146709222912 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954818048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 181444096 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚708396 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚939738 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚1629594 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 40735264768 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 866123776 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 2875281690112 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDev›q Õû iceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 3103168254464 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚708396 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚835 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10192158720 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 2834546425344 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚413 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 2875955288576 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚231030 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _step‚1 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 2834564428288 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 866123776 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚414102 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚921198 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚843 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22628 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚231480 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 40583250658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚854 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0id…'¤ÿN ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 43352637440 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 2760906955776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845965 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚843 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863047680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10192158720 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22901 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 3146520891904 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚414100 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚708398 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚708398 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 % _timestamp‚1.7884449252427642e+09 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚939738 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚921198 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22628 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚413 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954818048 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚645130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 43517028352 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 2760889916416 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚645582 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 332015104 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 41237680866 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10192158720 trainer/global_step‚49 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 40735264768 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1648136 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 41390860288 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 3146709222912 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954818048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 181444096 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚708396 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚939738 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚1629594 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 40735264768 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 866123776 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 2875281690112 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 3103168254464 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚708396 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚835 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10192158720 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 2834546425344 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚413 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 2875955288576 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚231030 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _step‚1 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 2834564428288 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 866123776 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚414102 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚921198 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚843 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22628 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚231480 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 40583250658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚854 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 2875955288576 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863047680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚843 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚ 40.7145904 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 2834564428288 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 2834546425344 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 3103192194560 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 164390912 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak"trainer/global_step2:JPÞÞªVn¨bi FDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed"trainer/global_step2:JPýê˜Kl©bg DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak"trainer/global_step2:JPª^Ÿqªbl IDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated"trainer/global_step2:JPqT6s«bn KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed"trainer/global_step2:JP ´@j¬be BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated"trainer/global_step2:JPþ>dd­b_ DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current"trainer/global_step2:JP<³ýtf¯ba >DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed"trainer/global_step2:JPÔåÄ_°bZ 7DeviceStatsMonitor.on_train_batch_end/segment.all.freed"trainer/global_step2:JPY¾ƒk±bf CDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated"trainer/global_step2:JPÛ1[`²b[ 8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections"trainer/global_step2:JPÑyñÝg³bb ?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current"trainer/global_step2:JP[ó›e´b` =DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed"trainer/global_step2:JPF³¡hµbc @DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current"trainer/global_step2:JPv{7?j¶be BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated"trainer/global_step2:JP#ÍJe·b` =DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak"trainer/global_step2:JPÚ-™w¸br ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated"trainer/global_step2:JPvúšWj¹be BDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current"trainer/global_step2:JPMÓnXqºbl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated"trainer/global_step2:JP,ÌÙk»bf CDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current"trainer/global_step2:JPJìQ\¼bW 4DeviceStatsMonitor.on_train_batch_end/max_split_size"trainer/global_step2:JPÜ“T p½bk HDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current"trainer/global_step2:JP"÷‚e¾b` =DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak"trainer/global_step2:JP ¨b¿b] :DeviceStatsMonitor.on_train_batch_end/active.all.allocated"trainer/global_step2:JPè²(ãnÀbi FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed"trainer/global_step2:JPþ¶v}eÁb` =DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak"trainer/global_step2:JP"ÚÖmÂbh EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak"trainer/global_step2:JPÙZ‡lÃbg DDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak"trainer/global_step2:JPE,“nÄbi FDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current"trainer/global_step2:JPµ«šärÅbm JDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak"trainer/global_step2:JP¡chçhÆbc @DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed"trainer/global_step2:JPͨÈ%hÇbc @DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated"trainer/global_step2:JPtz>tmÈbh EDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current"trainer/global_step2:JPr©¬lÉbg DDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak"trainer/global_step2:JP昶&bÊb] :DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams"trainer/global_step2:JPœ\¶eËb` =DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak"trainer/global_step2:JP“oWÕfÌba >DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed"trainer/global_step2:JP ×<¸oÍbj GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated"trainer/global_step2:JPç²LpÎbk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current"trainer/global_step2:JP]¹ÌœhÏbc @DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current"trainer/global_step2:JPÏ DêsÐbn KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed"trainer/global_step2:JP‰N$]ÑbX 5DeviceStatsMonitor.on_train_batch_end/active.all.peak"trainer/global_step2:JP1|”ždÒb_ DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed"trainer/global_step2:JPÒëøÊfìba >DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated"trainer/global_step2:JPÐ"íhíbc @DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak"trainer/global_step2:JP1K+kkîbf CDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak"trainer/global_step2:JP1úófïba >DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed"trainer/global_step2:JP“9}wðbr ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated"trainer/global_step2:JPE3ô|mñbh EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed"trainer/global_step2:JP|Ft¢còb^ ;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated"trainer/global_step2:JP¬ÝmZ^óbY 6DeviceStatsMonitor.on_train_batch_end/segment.all.peak"trainer/global_step2:JPä\Bìuôbp MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current"trainer/global_step2:JP¡TÜdõb_ æ0lùbg DDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated"trainer/global_step2:JPBRjPmúbh EDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak"trainer/global_step2:JPež*jûbe BDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak"trainer/global_step2:JP ˜©füba >DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak"trainer/global_step2:JPæŽQ>oýbj GDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current"trainer/global_step2:JPK(½òpþbk HDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current"trainer/global_step2:JP\-A’qÿbl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated"trainer/global_step2:JPô¹} d€b_ u†bp MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current"trainer/global_step2:JP•ìg‡bb ?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak"trainer/global_step2:JPýKºbˆb] :DeviceStatsMonitor.on_train_batch_end/allocation.all.freed"trainer/global_step2:JP§CÈm‰bh EDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated"trainer/global_step2:JP÷=F~mŠbh EDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated"trainer/global_step2:JPš"Tj‹be BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated"trainer/global_step2:JPWeèômŒbh EDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed"trainer/global_step2:JP#šl±ibd ADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated"trainer/global_step2:JP0þv#nŽbi FDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed"trainer/global_step2:JP¼ë*&hbc @DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current"trainer/global_step2:JP=ZÖibd ADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current"trainer/global_step2:JPôÅm‘bh EDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed"trainer/global_step2:JPî´óùk’bf CDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed"trainer/global_step2:JPÚw-vV“bQ .DeviceStatsMonitor.on_train_batch_end/num_ooms"trainer/global_step2:JPQ¬‡Hr”bm JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated"trainer/global_step2:JPº’Í9h•bc @DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current"trainer/global_step2:JP/0>Ùe–b` =DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed"trainer/global_step2:JPèÐßóp—bk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current"trainer/global_step2:JP™-Ök˜bf CDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated"trainer/global_step2:JP#q/ki™bd ADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current"trainer/global_step2:JPøVEŸmšbh EDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current"trainer/global_step2:JPaÒ>§_›bZ 7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries"trainer/global_step2:JP/Té]œbX 5DeviceStatsMonitor.on_train_batch_end/num_device_free"trainer/global_step2:JPo{Ìérbm JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated"trainer/global_step2:JP OÈ[ižbd ADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed"trainer/global_step2:JPÉ4oŸbj GDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current"trainer/global_step2:JPÄOYk. ) _wandbruntime‚41  _runtime‚41Kê¹"¡‡M EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚657898 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 2933127768576 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚413 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚863 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 3164842932224 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 866123776 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚413 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1680632 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 41554692608 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 192512000 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 2933127768576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10192158720 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 2890917467136 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 44205270528 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚843 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 2890899464192 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚722412 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚843 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚413 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚1662064 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚958218 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚422270 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 3209216774656 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 2857193732598 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 42054116038 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 866123776 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚939652 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 42210301440 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1680632 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 332015104 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 44373842432 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚843 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚843 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 168571É‹¥k904 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 41399685622 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚236085 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 3164818992128 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10192158720lñR!‘&¢‹M [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 168571904 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 41399685622 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚236085 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 3164818992128 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10192158720 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚657898 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 2933127768576 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚413 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚863 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 3164842932224 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 866123776 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚413 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1680632 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 41554692608 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 192512000 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 2933127768576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10192158720 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 2890917467136 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 44205270528 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚843 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 2890899464192 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚722412 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚843 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚413 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚1662064 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚958218 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚422270 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 3209216774656 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 2857193732598 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 42054116038 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 866123776 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚939652 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 42210301440 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1680632 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 332015104 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 44373842432 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚843 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚843¢”ÈÅC£b> loss/SRqqt1/assignment_loss"trainer/global_step2:JP‡ÃÓA¤b< loss/FBt4/assignment_loss"trainer/global_step2:JPWªSC¥b> loss/SRqqt4/assignment_loss"trainer/global_step2:JP ¨´@¦b; loss/FRt3/detection_loss"trainer/global_step2:JPÑ™Ï@§b; loss/FBt4/detection_loss"trainer/global_step2:JPõ¤A¨b< loss/FRt2/assignment_loss"trainer/global_step2:JP€'¿ @©b; loss/FRt4/detection_loss"trainer/global_step2:JP™ ª]Bªb= loss/SRqqt1/detection_loss"trainer/global_step2:JP++@«b; loss/FBt2/detection_loss"trainer/global_step2:JPןٖC¬b> loss/SRqqt2/assignment_loss"trainer/global_step2:JP9+[C­b> loss/SRqqt3/assignment_loss"trainer/global_step2:JPf¿ÓA®b< loss/FBt2/assignment_loss"trainer/global_step2:JPá±W§@¯b; loss/FRt2/detection_loss"trainer/global_step2:JPû\zrB°b= loss/SRqqt2/detection_loss"trainer/global_step2:JPÃ]‰#@±b; loss/FBt1/detection_loss"trainer/global_step2:JP\ã¨Ç@²b; loss/FRt1/detection_loss"trainer/global_step2:JPP…A³b< loss/FRt1/assignment_loss"trainer/global_step2:JPÏvgA´b< loss/FBt3/assignment_loss"trainer/global_step2:JP•¨ô@µb; loss/FBt3/detection_loss"trainer/global_step2:JPdµ ¢A¶b< loss/FRt4/assignment_loss"trainer/global_step2:JP’3$þA·b< loss/FBt1/assignment_loss"trainer/global_step2:JP¸ÿNö7¸b2 loss/total_loss"trainer/global_step2:JPAî-¹b( epoch"trainer/global_step2:JPˆ1°{Aºb< loss/FRt3/assignment_loss"trainer/global_step2:JP5{B»b= loss/SRqqt3/detection_loss"trainer/global_step2:JPþqUB¼b= loss/SRqqt4/detection_loss"trainer/global_step2:JP’>..½) _wandbruntime‚41  _runtime‚41ä°4C¾½ /loss/FRt2/assignment_loss‚7.249904568130906 .loss/FRt4/detection_loss‚4.622422016764942 1loss/SRqqt1/detection_loss‚3.3706575363874434 .loss/FBt2/detection_loss‚6.345970490230964 % _timestamp‚1.7884449259402936e+09 1loss/SRqqt2/assignment_loss‚5.392683811187744 1loss/SRqqt3/assignment_loss‚5.406883132073187 .loss/FBt2/assignment_loss‚2.76156132037823 .loss/FRt2/detection_loss‚4.465765608952859 0loss/SRqqt2/detection_loss‚3.312047416567802 /loss/FBt1/detection_loss‚11.730311145074666 .loss/FRt1/detection_loss‚3.641976148200532 /loss/FRt1/assignment_loss‚7.468651761611302 /loss/FBt3/assignment_loss‚2.769769637815414 .loss/FBt3/detection_loss‚5.736641488728985 .loss/FRt4/assignment_loss‚7.33589307885421 /loss/FBt1/assignment_loss‚2.735202878713608 &loss/total_loss‚120.84780347980009 epoch‚0 _runtime‚ 41.411829126 /loss/FRt3/assignment_loss‚7.208516233107623 0loss/SRqqt3/detection_loss‚2.714515046006249 1loss/SRqqt4/detection_loss‚3.6994999725032938 trainer/global_step‚49 1loss/SRqqt1/assignment_loss‚5.461341400146484 /loss/FBt4/assignment_loss‚2.769911215223115 1loss/SRqqt4/assignment_loss‚5.488041162490845 .loss/FRt3/detection_loss‚3.457760976810081 .loss/FBt4/detection_loss‚5.701875433839601 _step‚3oJÉG¿Á .loss/FRt4/assignment_loss‚7.33589307885421 /loss/FBt1/assignment_loss‚2.735202878713608 &loss/total_loss‚120.84780347980009 epoch‚0 _runtime‚ 41.411829126 /loss/FRt3/assignment_loss‚7.208516233107623 0loss/SRqqt3/detection_loss‚2.714515046006249 1loss/SRqqt4/detection_loss‚3.6994999725032938 trainer/global_step‚49 1loss/SRqqt1/assignment_loss‚5.461341400146484 /loss/FBt4/assignment_loss‚2.769911215223115 1loss/SRqqt4/assignment_loss‚5.488041162490845 .loss/FRt3/detection_loss‚3.457760976810081 .loss/FBt4/detection_loss‚5.701875433839601 _step‚3 /loss/FRt2/assignment_loss‚7.249904568130906 .loss/FRt4/detection_loss‚4.622422016764942 1loss/SRqqt1/detection_loss‚3.3706575363874434 .loss/FBt2/detection_loss‚6.345970490230964 % _timestamp‚1.7884449259402936e+09 1loss/SRqqt2/assignment_loss‚5.392683811187744 1loss/SRqqt3/assignment_loss‚5.406883132073187 .loss/FBt2/assignment_loss‚2.76156132037823 .loss/FRt2/detection_loss‚4.465765608952859 0loss/SRqqt2/detection_loss‚3.312047416567802 /loss/FBt1/detection_loss‚11.730311145074666 .loss/FRt1/detection_loss‚3.641976148200532 /loss/FRt1/assignment_loss‚7.468651761611302 /loss/FBt3/assignment_loss‚2.769769637815414 .loss/FBt3/detection_loss‚5.736641488728985ú…®RžÀ‚(:“ ‚æÔ§Ñçü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.502517700195312* gpu.0.memoryAllocatedBytes‚ 10739580928 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.905' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.580999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0E mŒ"Á‚(:— ‚æÔ£ä‘ÿ* disk./.usagePercent‚19.877394714955887 disk.drbd2080.in‚0 disk.sda1.out‚0$ disk./.usageGB‚296.1875457763672! disk.dm-0.out‚147.08837890625' proc.memory.availableMB‚ 1968.734375 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 cpu‚25.510345562831212 network.sent‚64251 network.recv‚75023 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.sda2.in‚0$ disk.nvme0n1.out‚147.08837890625 disk.dm-1.in‚0 disk.sda3.out‚2 disk.sda.in‚0 disk.sda1.in‚0$ memory_percent‚87.98379898071289 proc.cpu.threads‚17 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd1.in‚0" proc.memory.rssMB‚ 2550.8828125 disk.sda2.out‚0 disk.dm-1.out‚2& disk.nvme0c0n1.out‚147.08837890625 disk.rbd0.in‚0 disk.sda.out‚2 disk.rbd1.out‚ 0.3359375* proc.memory.percent‚15.569353103637695 disk.sda3.in‚0L‡lê“‚(:ˆ ‚æÔòóŸý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.502517700195312* gpu.0.memoryAllocatedBytes‚ 10739580928 gpu.0.temp‚38.0 gpu.0.powerWatts‚127.972' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.5944 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0#²v>Â(:³ ‚æÔ¢ëÿ$ memory_percent‚87.99748420715332 proc.cpu.threads‚17& disk.nvme0c0n1.out‚204.63818359375 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 0.4296875 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.sda3.in‚0 disk.sda.out‚ 2.73828125 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.drbd2265.out‚0 disk.sda2.in‚0 cpu‚25.43295108182462 disk.drbd2266.out‚0 network.recv‚109167* disk./.usagePercent‚19.879371349127755 disk.nvme0c0n1.in‚0.078125! disk.dm-0.out‚204.63818359375 disk.sda2.out‚0( proc.memory.availableMB‚ 1966.4921875* proc.memory.percent‚15.569400787353516$ disk./.usageGB‚296.2169990539551 disk.drbd2080.in‚0$ disk.nvme0n1.out‚204.63818359375 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.rbd0.out‚0 network.sent‚134826 disk.drbd2080.out‚0 disk.sda3.out‚ 2.73828125 disk.drbd1716.in‚0! proc.memory.rssMB‚ 2550.890625 disk.rbd0.in‚0 disk.sda.in‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 2.73828125ã´lVžÄ‚(:“ Ÿ‚æÔ¨·¦ý gpu.0.gpu‚49.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.502517700195312* gpu.0.memoryAllocatedBytes‚ 10739580928 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.935' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.186999999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0\ìÐAÅ‚(:¶ Ÿ‚æÔ’ýÑÿ* proc.memory.percent‚15.569400787353516* disk./.usagePercent‚19.880096874479793 disk.drbd1716.out‚0 proc.cpu.threads‚17 disk.rbd1.out‚ 0.6171875 disk.dm-1.out‚ 3.16796875 cpu‚25.375174487517672 disk.rbd1.in‚0 network.sent‚140680 disk.nvme0c0n1.in‚0.078125& disk.nvme0c0n1.out‚255.04443359375) proc.memory.availableMB‚ 1967.44921875 disk.sda1.in‚0 disk.sda2.out‚0% disk./.usageGB‚296.22780990600586 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0$ memory_percent‚87.99164295196533 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0$ disk.nvme0n1.out‚255.04443359375 disk.sda.out‚ 3.16796875 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 3.16796875 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 network.recv‚113859! disk.dm-0.out‚255.04443359375 disk.dm-1.in‚0! proc.memory.rssMB‚ 2550.890625µï.Æ) _wandbruntime‚77  _runtime‚77šb‰YÂǼ ' lr-AdamW/pg1‚1.0373766404261373e-05 ' lr-AdamW/pg2‚1.0373766404261373e-05 trainer/global_step‚99 % _timestamp‚1.7884449622471113e+09 _runtime‚ 77.71882609 _step‚4Â}÷¬ÆÈÀ ' lr-AdamW/pg1‚1.0373766404261373e-05 ' lr-AdamW/pg2‚1.0373766404261373e-05 trainer/global_step‚99 % _timestamp‚1.7884449622471113e+09 _runtime‚ 77.71882609 _step‚4Q1æ/.É) _wandbruntime‚77  _runtime‚77õÄ.R#ʇO E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 5734593036800 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 5506096444416 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 5734593036800 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3274988 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1847250 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1281722 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚459222 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1847250 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1865790 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚413 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 81721070592 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚843 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚413 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 6271899681792 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10192158720 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3256446 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 82376666112 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 151808000 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1409196 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚843 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 82073020274 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1409196 % _timestamp‚1.7884449622479672e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 5733919438336 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 81721070592 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10192158720 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 86346352128 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 5506113483776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 5652198367744 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1409198 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 866123776 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 5652198367744 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 86194544128 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1865790 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 6185729077760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚843 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 866123776 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚843 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 5652216370688 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 82376666112 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚413 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 866123776 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚439 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 Hoê;DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1409196 % _timestamp‚1.7884449622479672e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 5733919438336 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 81721070592 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10192158720 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 86346352128 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 5506113483776 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 5652198367744 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1409198 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 866123776 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 5652198367744 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 86194544128 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1865790 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 6185729077760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚843 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 866123776 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚843 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 5652216370688 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 82376666112 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚413 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 866123776 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚439 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 HDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3307484 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1884270 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 82540498432 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚453 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 5643235600006 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 866123776 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 192512000 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚843 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1865704 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚863 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚464205 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 6247355875328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚843 $ _timestamp‚1.788444962961788e+09 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1884270 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1294875 _runtime‚ 78.434028373 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚413 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 82540498432 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1865704 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1423214 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1423212 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 82235025030 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 5791765516800 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚843 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 6334403763200 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1423214 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 5561017614336 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚3288916 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 5708551406592 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚413 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 83196107264 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 5791765516800 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚830670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚830668 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 332015104 _step‚6 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1423212 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 5791091905024 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1294420 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 5708551406592 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚463752 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 6247379815424 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 87216459776 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 866123776 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 168571904 trainer/global_step‚99 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3307484 N8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10192158720 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 5561000574976 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 82889455446 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 5708569409536 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 183540224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚843 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3307484 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1884270 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 82540498432 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚453 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 5643235600006 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 866123776 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 192512000 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚843 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1865704 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚863 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚464205 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 6247355875328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚843 $ _timestamp‚1.788444962961788e+09 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1884270 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1294875 _runtime‚ 78.434028373 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚413 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 82540498432 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1865704 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1423214 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1423212 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 82235025030 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 5791765516800 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚843 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 6334403763200 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1423214 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 5561017614336 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚3288916 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 5708551406592 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚413 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 83196107264 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 5791765516800 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚830670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚830668 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 332015104 _step‚6 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1423212 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 5791091905024 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1294420 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 5708551406592 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚463752 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 6247379815424 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 87216459776 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 866123776 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 168571904 trainer/global_step‚99 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3307484 N8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10192158720 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 5561000574976 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 82889455446 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 5708569409536 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 183540224 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚843 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 Fl˜Ã4Ó‚(:© ®‚æÔ󖘀 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.rbd0.out‚0 disk.sda3.out‚3.859375 disk.drbd2080.in‚0$ memory_percent‚87.99991607666016% disk.nvme0c0n1.out‚315.2548828125 disk.sda2.in‚0# disk.nvme0n1.out‚315.2548828125 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0* proc.memory.percent‚15.569448471069336 disk.rbd0.in‚0 disk.rbd1.out‚ 0.6171875 network.recv‚120136 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.dm-1.in‚0 network.sent‚148493) disk./.usagePercent‚19.88258501170507% disk./.usageGB‚296.26488494873047 disk.sda.out‚3.859375 disk.rbd1.in‚0 disk.drbd1716.out‚0& proc.memory.availableMB‚ 1966.09375" proc.memory.rssMB‚ 2550.8984375 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.sda.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17 disk.drbd2266.out‚0 disk.dm-0.out‚315.2548828125 disk.dm-1.out‚3.859375 cpu‚25.33402077101604‡8ß•žÔ‚(:“ ½‚æÔ£¯¥ý gpu.0.gpu‚32.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.502517700195312* gpu.0.memoryAllocatedBytes‚ 10739580928 gpu.0.temp‚37.0 gpu.0.powerWatts‚129.241' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.848200000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¹Y¥ü:Õ‚(:¯ ½‚æÔÚÍÑÿ disk.nvme0n1.in‚0.078125) disk./.usagePercent‚19.88162114333866 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.in‚0$ memory_percent‚88.00234794616699 proc.memory.rssMB‚ 2550.90625 disk.sda3.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚359.2119140625 disk.sda3.out‚ 4.22265625 disk.sda.in‚0 disk.sda1.out‚0 disk.drbd2080.out‚0 cpu‚25.302494040406295 proc.cpu.threads‚17$ disk./.usageGB‚296.2505226135254 disk.rbd0.out‚0 disk.sda.out‚ 4.22265625 disk.sda2.out‚0 disk.rbd1.out‚ 0.6796875 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.dm-0.out‚359.2119140625 disk.drbd2265.in‚0( proc.memory.availableMB‚ 1965.6953125 network.sent‚197672 disk.rbd0.in‚0 disk.rbd1.in‚0# disk.nvme0n1.out‚359.2119140625 disk.dm-1.out‚ 4.22265625* proc.memory.percent‚15.569496154785156 network.recv‚125034 disk.drbd2266.in‚0 disk.drbd2266.out‚0?lÁ‡0Ö+ _wandbruntime‚115  _runtime‚115â•€ƒÃ×½ _runtime‚ 115.303116186 _step‚8 & lr-AdamW/pg1‚1.561304236600954e-05 & lr-AdamW/pg2‚1.561304236600954e-05 trainer/global_step‚149 % _timestamp‚1.7884449998314562e+09ª=úâÇØÁ _runtime‚ 115.303116186 _step‚8 & lr-AdamW/pg1‚1.561304236600954e-05 & lr-AdamW/pg2‚1.561304236600954e-05 trainer/global_step‚149 % _timestamp‚1.7884449998314562e+09ï õ¿0Ù+ _wandbruntime‚115  _runtime‚115ImE¼™'Ú“O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 9397364873728 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2109996 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚4883298 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 122908359682 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 8593230785024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 190428160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 8469850310144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 8469850310144 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 8251320011776 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚4901840 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 9268242020864 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 8592557186560 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 8251302972416 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚687447 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1918787 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2109998 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2791842 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2109998 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 8593230785024 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 122706876416 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚687885 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 129122852864 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2791842 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 % _timestamp‚1.7884449998322592e+09 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2109996 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 166488064 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1230902 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚855 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚9 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1230900 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 122253929474 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 129289340928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0•]í'Û—O ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 8469868313088 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚4883298 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 122908359682 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚ 8593230785024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 190428160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 8469850310144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 8469850310144 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 8251320011776 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚4901840 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 9268242020864 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 8592557186560 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 8251302972416 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚687447 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1918787 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2109998 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚2791842 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2109998 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 8593230785024 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 122706876416 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚687885 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 129122852864 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2791842 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 % _timestamp‚1.7884449998322592e+09 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2109996 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 166488064 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1230902 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚855 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚9 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1230900 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 122253929474 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 129289340928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 9397364873728 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2109996 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2124012 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1930972 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2124012 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 9459866802688 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172766208 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2124014 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚855 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 9329892758528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 8526221351936 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2791756 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1239068 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 196706304 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2810322 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 8650403265024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 130146810368 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚692342 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2791756 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4915768 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 $ _timestamp‚1.788445000509862e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 8526203348992 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 8526203348992 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 123526304256 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚438 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4915768 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 8526221351936 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 124181913088 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 8649729653248 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1239070 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 8306224142336 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2124014 _runtime‚ 115.981889591 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 123526304256 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 NDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4934336 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 8306207102976 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 123070364438 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚440 trainer/global_step‚149 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4934336’ße§&Þ¡M TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚692342 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2791756 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4915768 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 $ _timestamp‚1.788445000509862e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 8526203348992 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 8526203348992 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 123526304256 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚438 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4915768 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 8526221351936 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 124181913088 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 8649729653248 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1239070 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 8306224142336 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2124014 _runtime‚ 115.981889591 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 123526304256 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 NDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4934336 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 8306207102976 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 123070364438 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚440 trainer/global_step‚149 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4934336 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚874 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2124012 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1930972 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2124012 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 9459866802688 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172766208 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2124014 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚855 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 9329892758528 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 8526221351936 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2791756 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1239068 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 196706304 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2810322 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 8650403265024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 130146810368 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ð0’0ß+ _wandbruntime‚115  _runtime‚115¡ÞrLàÆ 0loss/FRt1/assignment_loss‚7.3287877411138815 1loss/SRqqt3/detection_loss‚3.9179416143081407 0loss/SRqqt4/detection_loss‚5.039599094320746 &loss/total_loss‚133.12746331616495 0loss/FBt2/assignment_loss‚2.4164826436476274 -loss/FBt4/detection_loss‚2.27711648296323 trainer/global_step‚149 1loss/SRqqt1/assignment_loss‚5.530318796634674 /loss/FRt1/detection_loss‚2.8661464235333147 .loss/FRt4/detection_loss‚5.130963557958603 /loss/FBt1/detection_loss‚11.362354466691613 /loss/FBt3/detection_loss‚13.233439630695752 .loss/FRt2/detection_loss‚5.246284861458537 /loss/FRt3/assignment_loss‚7.167990947591847 2loss/SRqqt2/assignment_loss‚5.3559792479690245 0loss/SRqqt3/assignment_loss‚5.37321598963304 .loss/FBt2/detection_loss‚15.40256038850004 $ _timestamp‚1.788445000511278e+09 .loss/FBt4/assignment_loss‚2.44225728053313 _runtime‚ 115.982968056 /loss/FRt2/assignment_loss‚7.131610306826505 1loss/SRqqt4/assignment_loss‚5.490154448677512 0loss/FBt1/assignment_loss‚2.3671657145023346 0loss/FBt3/assignment_loss‚2.4285597630909512 /loss/FRt3/detection_loss‚2.6725930702583542 epoch‚0 _step‚11 /loss/FRt4/assignment_loss‚7.368357535770961 0loss/SRqqt1/detection_loss‚2.274200922518503 1loss/SRqqt2/detection_loss‚3.3033823869666277¨ucPáÊ 0loss/FBt2/assignment_loss‚2.4164826436476274 -loss/FBt4/detection_loss‚2.27711648296323 trainer/global_step‚149 1loss/SRqqt1/assignment_loss‚5.530318796634674 /loss/FRt1/detection_loss‚2.8661464235333147 .loss/FRt4/detection_loss‚5.130963557958603 /loss/FBt1/detection_loss‚11.362354466691613 /loss/FBt3/detection_loss‚13.233439630695752 .loss/FRt2/detection_loss‚5.246284861458537 /loss/FRt3/assignment_loss‚7.167990947591847 2loss/SRqqt2/assignment_loss‚5.3559792479690245 0loss/SRqqt3/assignment_loss‚5.37321598963304 .loss/FBt2/detection_loss‚15.40256038850004 $ _timestamp‚1.788445000511278e+09 .loss/FBt4/assignment_loss‚2.44225728053313 _runtime‚ 115.982968056 /loss/FRt2/assignment_loss‚7.131610306826505 1loss/SRqqt4/assignment_loss‚5.490154448677512 0loss/FBt1/assignment_loss‚2.3671657145023346 0loss/FBt3/assignment_loss‚2.4285597630909512 /loss/FRt3/detection_loss‚2.6725930702583542 epoch‚0 _step‚11 /loss/FRt4/assignment_loss‚7.368357535770961 0loss/SRqqt1/detection_loss‚2.274200922518503 1loss/SRqqt2/detection_loss‚3.3033823869666277 0loss/FRt1/assignment_loss‚7.3287877411138815 1loss/SRqqt3/detection_loss‚3.9179416143081407 0loss/SRqqt4/detection_loss‚5.039599094320746 &loss/total_loss‚133.12746331616495 P|H?’â‚(:‡ Ì‚æÔš„þü gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚171.711' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.3422 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0«”Ü5ã‚(:ª Ì‚æÔì¾”€ disk.sda.in‚0$ disk./.usageGB‚296.2603187561035 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0& disk.nvme0c0n1.out‚413.81494140625$ memory_percent‚88.01140785217285 disk.dm-0.in‚0.078125 network.recv‚130604 disk.sda3.in‚0 disk.sda3.out‚4.96875 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.drbd2080.out‚0( proc.memory.availableMB‚ 1964.2109375 disk.drbd2266.in‚0 disk.dm-1.in‚0 disk.dm-1.out‚4.96875 disk.drbd2265.in‚0 cpu‚25.276930005726136 proc.cpu.threads‚17 network.sent‚204573* disk./.usagePercent‚19.882278570686587 disk.sda1.out‚0 disk.sda2.in‚0 disk.rbd1.out‚ 0.6796875 disk.drbd2265.out‚0 disk.sda.out‚4.96875 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0! disk.dm-0.out‚413.81494140625$ disk.nvme0n1.out‚413.81494140625 proc.memory.rssMB‚ 2550.90625* proc.memory.percent‚15.569496154785156¼³Õñ“ä‚(:ˆ Û‚æÔ´¡‚ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚174.452' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.8904 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0e3ͧ@å‚(:µ Û‚æÔТ›ÿ disk.dm-1.out‚ 5.41796875 cpu‚25.25650105138418 disk.sda.out‚ 5.41796875 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.drbd2266.in‚0! disk.dm-0.out‚456.49462890625 disk.sda1.out‚0 disk.drbd2265.out‚0 network.recv‚135358$ disk./.usageGB‚296.2676124572754 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.sda2.in‚0$ disk.nvme0n1.out‚456.49462890625 disk.dm-0.in‚0.078125( proc.memory.availableMB‚ 1963.6953125 disk.dm-1.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 0.8369140625$ memory_percent‚88.01455497741699 proc.cpu.threads‚17 disk.sda3.out‚ 5.41796875 proc.memory.rssMB‚ 2550.90625 disk.rbd0.out‚0 disk.sda.in‚0 network.sent‚253780 disk.drbd2266.out‚0 disk.rbd1.in‚0* proc.memory.percent‚15.569496154785156 disk.drbd1716.out‚0& disk.nvme0c0n1.out‚456.49462890625* disk./.usagePercent‚19.882768057092058­2—žæ‚(:“ ê‚æÔ„­›þ gpu.0.gpu‚38.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.834' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.766799999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¸ãdAç‚(:¶ ê‚æÔÛ¾Œ€ cpu‚25.238956466620213 disk.rbd1.in‚0 disk.dm-1.out‚ 6.46484375 disk.drbd2080.out‚0& disk.nvme0c0n1.out‚526.21435546875 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 6.46484375 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0) proc.memory.availableMB‚ 1963.61328125 proc.memory.rssMB‚ 2550.90625 disk.drbd1716.in‚0 network.recv‚141114 disk.sda.in‚0 disk.sda1.out‚0 disk.sda3.out‚ 6.46484375* proc.memory.percent‚15.569496154785156 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 0.9228515625 disk.drbd2265.in‚0 disk.sda2.in‚0# memory_percent‚88.0150556564331 network.sent‚260913 disk.rbd0.in‚0$ disk.nvme0n1.out‚526.21435546875 disk.drbd2080.in‚0 proc.cpu.threads‚17 disk.dm-1.in‚0* disk./.usagePercent‚19.883118019391365$ disk./.usageGB‚296.2728271484375 disk.rbd0.out‚0! disk.dm-0.out‚526.21435546875 disk.nvme0n1.in‚0.078125 disk.sda3.in‚0 disk.sda1.in‚0 disk.drbd2266.out‚0Í9K0è+ _wandbruntime‚152  _runtime‚152DµÜÆéÀ trainer/global_step‚199 % _timestamp‚1.7884450371141224e+09 _runtime‚ 152.585684273 _step‚12 ' lr-AdamW/pg1‚2.0852318327757708e-05 ' lr-AdamW/pg2‚2.0852318327757708e-05Ç%ÊêÄ _runtime‚ 152.585684273 _step‚12 ' lr-AdamW/pg1‚2.0852318327757708e-05 ' lr-AdamW/pg2‚2.0852318327757708e-05 trainer/global_step‚199 % _timestamp‚1.7884450371141224e+09 ¯Ð²ð0ë+ _wandbruntime‚152  _runtime‚152 ¬C‘ª'ì¤O VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3699354 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚11159598769298 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 163743699090 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2810798 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 163692682240 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚11451868533248 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚916481 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2810798 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 171953641984 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2555331 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚916031 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚855 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚12522732546048 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 163089268882 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 163692682240 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚12350802844160 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚10996526539776 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚199 % _timestamp‚1.7884450371148143e+09 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 172118032896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚11287502252544 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2810796 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3717894 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1639300 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3699354 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚11451194934784 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚ 152.586589812 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚12522920877056 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2555783 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚13 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚11451868533248 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚874 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚11160270238866 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 164348277760 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2810796 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚12350778904064 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3717894 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚11287520255488 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚6510150 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚11451194934784 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 164348277760 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360@ƒ=“ë í¨O D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚6510150 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚11451194934784 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 164348277760 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3699354 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚11159598769298 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 163743699090 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2810798 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 163692682240 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚11451868533248 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚916481 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 WJDeviceStatsMonitor.on_train_batch_start/allocat´Û6EÃed_bytes.large_pool.current‚18002944 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2810798 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 171953641984 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2555331 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚916031 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚855 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚12522732546048 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 163089268882 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 163692682240 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚12350802844160 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚10996526539776 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚199 % _timestamp‚1.7884450371148143e+09 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 172118032896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚11287502252544 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2810796 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3717894 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1639300 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3699354 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚11451194934784 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚ 152.586589812 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚12522920877056 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2555783 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚13 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚11451868533248 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚874 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚11160270238866 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 164348277760 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2810796 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚12350778904064 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3717894 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚11287520255488 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 Ñ@»G0î+ _wandbruntime‚153  _runtime‚153± Áj´&ï®M J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚920605 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 trainer/global_step‚199 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2568533 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚460 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1647470 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1647468 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3717808 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚11343855291392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚11509041013248 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚11343855291392 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2824814 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 163905703846 _step‚14 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚11509041013248 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3736374 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6542620 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172766208 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚11215319334822 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3717808 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 164560134262 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6561188 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2568073 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚12585235206144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2824814 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚11215990804598 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚855 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚12585431912448 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 196706304 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚ 153.301188731 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6561188 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚921063 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 172805564416 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3736374 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 172978330624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 165167718912 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚458 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0Y ¸&ð²M PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3717808 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 164560134262 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6561188 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2568073 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚12585235206144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2824814 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚11215990804598 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚855 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚12585431912448 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 196706304 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚ 153.301188731 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6561188 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚921063 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 172805564416 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3736374 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 172978330624 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 165167718912 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚458 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚920605 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 trainer/global_step‚199 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2568533 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚460 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1647470 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1647468 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3717808 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚11343855291392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚11509041013248 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚11343855291392 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2824814 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 163905703846 _step‚14 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚11509041013248 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚3736374 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6542620 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172766208 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚11215319334822 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928ˆª^0ñ+ _wandbruntime‚153  _runtime‚153Ì;îIòà &loss/total_loss‚128.48746686142084 0loss/FBt1/assignment_loss‚1.4019795060157776 /loss/FBt2/detection_loss‚11.098452419042587 epoch‚0 1loss/SRqqt3/assignment_loss‚5.284918767434579 /loss/FRt1/detection_loss‚3.9147191108627752 0loss/SRqqt3/detection_loss‚2.954110178820513 % _timestamp‚1.7884450378308816e+09 /loss/FRt1/assignment_loss‚7.452666423537514 /loss/FRt3/assignment_loss‚7.163879132270813 .loss/FRt2/detection_loss‚7.430636597307105 1loss/SRqqt2/assignment_loss‚5.358664035797119 .loss/FBt4/assignment_loss‚1.45584499013835 /loss/FBt3/detection_loss‚13.978960078496199 0loss/FBt2/assignment_loss‚1.6297333717346192 0loss/SRqqt1/detection_loss‚2.139094837512525 .loss/FBt4/detection_loss‚1.806396365903342 trainer/global_step‚199 _runtime‚ 153.302401661 /loss/FRt2/assignment_loss‚7.187647744228966 .loss/FRt3/detection_loss‚4.171102833095938 -loss/FRt4/detection_loss‚4.15670973956585 0loss/SRqqt4/detection_loss‚8.166114293393635 .loss/FBt1/detection_loss‚8.311177562583577 _step‚15 1loss/SRqqt4/assignment_loss‚5.448728833879743 0loss/FBt3/assignment_loss‚1.6196844852887666 /loss/FRt4/assignment_loss‚7.327166092395783 0loss/SRqqt1/assignment_loss‚5.46775804247175 1loss/SRqqt2/detection_loss‚3.5613214196430314aþ*0MóÇ 1loss/SRqqt3/assignment_loss‚5.284918767434579 /loss/FRt1/detection_loss‚3.9147191108627752 0loss/SRqqt3/detection_loss‚2.954110178820513 % _timestamp‚1.7884450378308816e+09 /loss/FRt1/assignment_loss‚7.452666423537514 /loss/FRt3/assignment_loss‚7.163879132270813 .loss/FRt2/detection_loss‚7.430636597307105 1loss/SRqqt2/assignment_loss‚5.358664035797119 .loss/FBt4/assignment_loss‚1.45584499013835 /loss/FBt3/detection_loss‚13.978960078496199 0loss/FBt2/assignment_loss‚1.6297333717346192 0loss/SRqqt1/detection_loss‚2.139094837512525 .loss/FBt4/detection_loss‚1.806396365903342 trainer/global_step‚199 _runtime‚ 153.302401661 /loss/FRt2/assignment_loss‚7.187647744228966 .loss/FRt3/detection_loss‚4.171102833095938 -loss/FRt4/detection_loss‚4.15670973956585 0loss/SRqqt4/detection_loss‚8.166114293393635 .loss/FBt1/detection_loss‚8.311177562583577 _step‚15 1loss/SRqqt4/assignment_loss‚5.448728833879743 0loss/FBt3/assignment_loss‚1.6196844852887666 /loss/FRt4/assignment_loss‚7.327166092395783 0loss/SRqqt1/assignment_loss‚5.46775804247175 1loss/SRqqt2/detection_loss‚3.5613214196430314 &loss/total_loss‚128.48746686142084 0loss/FBt1/assignment_loss‚1.4019795060157776 /loss/FBt2/detection_loss‚11.098452419042587 epoch‚0üØ&“ô‚(:ˆ ù‚æÔ™À£ý gpu.0.gpu‚42.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.661' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.7322 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚08ëȵEõ‚(:º ù‚æÔ½€£€ disk.drbd2266.in‚0 disk.dm-0.in‚0.078125) proc.memory.availableMB‚ 1962.63671875* proc.memory.percent‚15.569519996643066 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0! disk.dm-0.out‚576.20947265625 disk.rbd1.in‚0 disk.rbd1.out‚ 0.9853515625 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.sda.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125$ disk.nvme0n1.out‚576.20947265625 disk.dm-1.in‚0 disk.dm-1.out‚ 6.76171875 network.recv‚145191 disk.sda2.in‚0 disk.sda2.out‚0$ memory_percent‚88.02101612091064 network.sent‚266596& disk.nvme0c0n1.out‚576.20947265625 disk.sda3.in‚0 disk.sda1.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125# proc.memory.rssMB‚ 2550.91015625 proc.cpu.threads‚17% disk./.usageGB‚296.28189849853516 disk.drbd1716.out‚0 cpu‚25.22431852438627* disk./.usagePercent‚19.883726805307784 disk.sda3.out‚ 6.76171875 disk.sda.out‚ 6.76171875êú&“ö‚(:ˆ ˆƒæÔ™¬ûü gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.523' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.3046 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¸>&A#÷‚(:³ ˆƒæÔö»îÿ$ disk./.usageGB‚296.2882843017578 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125# proc.memory.rssMB‚ 2550.91015625 disk.drbd1716.in‚0 disk.sda3.in‚0 disk.sda3.out‚ 7.3515625# disk.nvme0n1.out‚627.7900390625 disk.dm-1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.dm-1.out‚ 7.3515625 disk.rbd1.in‚0 disk.drbd2265.in‚0* proc.memory.percent‚15.569519996643066 cpu‚25.210729548474195 proc.cpu.threads‚17* disk./.usagePercent‚19.884155361920104 disk.rbd1.out‚ 0.9853515625 disk.drbd2266.out‚0 disk.dm-0.out‚627.7900390625 disk.sda.out‚ 7.3515625# meméF‡bory_percent‚88.0239725112915 network.sent‚317154 network.recv‚151755 disk.sda1.in‚0) proc.memory.availableMB‚ 1962.15234375 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125% disk.nvme0c0n1.out‚627.7900390625Q^ÆT0ø+ _wandbruntime‚189  _runtime‚1894ÐOþÆùÀ ' lr-AdamW/pg1‚2.6091594289505876e-05 ' lr-AdamW/pg2‚2.6091594289505876e-05 trainer/global_step‚249 % _timestamp‚1.7884450741641665e+09 _runtime‚ 189.637396923 _step‚16®ŽL½ÊúÄ ' lr-AdamW/pg1‚2.6091594289505876e-05 ' lr-AdamW/pg2‚2.6091594289505876e-05 trainer/global_step‚249 % _timestamp‚1.7884450741641665e+09 _runtime‚ 189.637396923 _step‚16þCMï0û+ _wandbruntime‚189  _runtime‚1899or¬'ü¦O LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3194523 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚860 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2047702 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3511596 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 162293760 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚15433315787264 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 215005637120 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 214843343360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1146382 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 204678488064 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1146821 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8155544 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚879 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _runtime‚ 189.638564227 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8137002 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3511596 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3511598 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚14310506281472 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚14105172197888 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚13945640636706 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 % _timestamp‚1.7884450741666472e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚14309832683008 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3194082 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2047700 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚15648159130624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 203924608290 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844˜<#°'ýªO D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8137002 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3511596 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3511598 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚14310506281472 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚14105172197888 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚13945640636706 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 % _timestamp‚1.7884450741666472e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚14309832683008 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3194082 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2047700 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚15648159130624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 203924608290 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3194523 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚860 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2047702 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3511596 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 162293760 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚15433315787264 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 215005637120 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 214843343360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1146382 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 204678488064 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1146821 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8155544 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚879 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _runtime‚ 189.638564227 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚00LB#0þ+ _wandbruntime‚190  _runtime‚190Š]¨ü¶&ÿ°M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3525614 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8188040 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4662426 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1150983 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚14161525236736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚879 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1151422 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4643860 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚14002032672006 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚15494990465024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚15494966524928 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3525612 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8188040 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8169472 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚13796637198336 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 205395473670 _runtime‚ 190.340196431 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 205497915904 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4643860 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3207292 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 % _timestamp‚1.7884450748679824e+09 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚14367005149696 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 206153524736 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2055868 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚13796620158976 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3206851 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 192512000 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 206153524736 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2055870 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 215693541376 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3525614 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 205497915904 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚15710660066304 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚18 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚14161507233792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚860 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 204741043254 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4662426 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚14161507233792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚249 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚14367678761472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 168571904 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚15710852578304 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0‹s$m€´M HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚15710852578304 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3525614 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8188040 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4662426 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1150983 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚14161525236736 UHDeviceStatsMonitor.on_train_batch_end/alí^}M located_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚879 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1151422 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚4643860 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚14002032672006 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚15494990465024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚15494966524928 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3525612 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8188040 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8169472 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚13796637198336 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 205395473670 _runtime‚ 190.340196431 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 205497915904 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4643860 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3207292 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 % _timestamp‚1.7884450748679824e+09 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚14367005149696 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 206153524736 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2055868 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚13796620158976 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3206851 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 192512000 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 206153524736 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2055870 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 215693541376 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3525614 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 205497915904 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚15710660066304 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚18 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚14161507233792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚860 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 204741043254 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4662426 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚14161507233792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚249 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚14367678761472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 168571904éGƒ0+ _wandbruntime‚190  _runtime‚190+Ü‚ZJ‚Ä 0loss/FRt2/assignment_loss‚7.2417803244157275 1loss/SRqqt4/assignment_loss‚4.757022510875355 1loss/SRqqt1/detection_loss‚2.1762749993475152 0loss/SRqqt4/detection_loss‚7.590888867324049 &loss/total_loss‚114.24920570630269 _runtime‚ 190.341198302 .loss/FBt2/assignment_loss‚0.51412916810889 /loss/FRt3/detection_loss‚3.3443872540568313 % _timestamp‚1.7884450748697054e+09 _step‚19 0loss/SRqqt3/detection_loss‚2.342315662361216 1loss/SRqqt3/assignment_loss‚4.502577178180218 .loss/FRt2/detection_loss‚5.459987836805257 .loss/FRt4/detection_loss‚3.359767729640007 1loss/SRqqt2/detection_loss‚3.4752318080752453 .loss/FBt2/detection_loss‚8.696195155382156 -loss/FBt3/detection_loss‚9.00945839881897 .loss/FBt4/detection_loss‚1.254005963495946 -loss/FRt1/detection_loss‚3.28150434519809 /loss/FRt3/assignment_loss‚7.147514472405116 /loss/FRt4/assignment_loss‚7.338413515090942 1loss/SRqqt1/assignment_loss‚4.876988792419434 0loss/FBt1/assignment_loss‚0.8038346962286875 epoch‚0 /loss/FBt1/detection_loss‚14.135395467281342 trainer/global_step‚249 0loss/FBt3/assignment_loss‚0.4259731497615576 /loss/FRt1/assignment_loss‚7.472551877682026 1loss/SRqqt2/assignment_loss‚4.669496647855069 0loss/FBt4/assignment_loss‚0.3735098854930543™ïˆGNƒÈ /loss/FBt1/detection_loss‚14.135395467281342 trainer/global_step‚249 0loss/FBt3/assignment_loss‚0.4259731497615576 /loss/FRt1/assignment_loss‚7.472551877682026 1loss/SRqqt2/assignment_loss‚4.669496647855069 0loss/FBt4/assignment_loss‚0.3735098854930543 0loss/FRt2/assignment_loss‚7.2417803244157275 1loss/SRqqt4/assignment_loss‚4.757022510875355 1loss/SRqqt1/detection_loss‚2.1762749993475152 0loss/SRqqt4/detection_loss‚7.590888867324049 &loss/total_loss‚114.24920570630269 _runtime‚ 190.341198302 .loss/FBt2/assignment_loss‚0.51412916810889 /loss/FRt3/detection_loss‚3.3443872540568313 % _timestamp‚1.7884450748697054e+09 _step‚19 0loss/SRqqt3/detection_loss‚2.342315662361216 1loss/SRqqt3/assignment_loss‚4.502577178180218 .loss/FRt2/detection_loss‚5.459987836805257 .loss/FRt4/detection_loss‚3.359767729640007 1loss/SRqqt2/detection_loss‚3.4752318080752453 .loss/FBt2/detection_loss‚8.696195155382156 -loss/FBt3/detection_loss‚9.00945839881897 .loss/FBt4/detection_loss‚1.254005963495946 -loss/FRt1/detection_loss‚3.28150434519809 /loss/FRt3/assignment_loss‚7.147514472405116 /loss/FRt4/assignment_loss‚7.338413515090942 1loss/SRqqt1/assignment_loss‚4.876988792419434 0loss/FBt1/assignment_loss‚0.8038346962286875 epoch‚0ÇâV“„‚(:ˆ —ƒæÔûϳý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.192' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.8384 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ù:©*B…‚(:· —ƒæÔ¦ü¶ÿ$ disk.nvme0n1.out‚659.69482421875 disk.drbd2265.out‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0$ disk./.usageGB‚296.2985305786133 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.rbd1.out‚ 1.142578125 disk.drbd1716.in‚0 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.sda.in‚0 disk.sda3.out‚ 7.73828125 disk.rbd0.out‚0! disk.dm-0.out‚659.69482421875 disk.sda1.out‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 7.73828125# proc.memory.rssMB‚ 2550.91015625* proc.memory.percent‚15.569519996643066& disk.nvme0c0n1.out‚659.69482421875* disk./.usagePercent‚19.884842998157072 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda2.in‚0' proc.memory.availableMB‚ 1962.484375 cpu‚25.201190708010074 network.recv‚155628 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 7.73828125 disk.nvme0n1.in‚0.078125$ memory_percent‚88.02194595336914 network.sent‚322820 disk.nvme0c0n1.in‚0.078125 disk.sda2.out‚0±(󞆂(:“ ¦ƒæԯЧý gpu.0.gpu‚22.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.017' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.203399999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 AiD‡‚(:¹ ¦ƒæÔ—Œ¢ÿ$ memory_percent‚88.02220821380615 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 proc.cpu.threads‚17 disk.dm-1.in‚0 disk.drbd1716.out‚0 network.sent‚372561 disk.sda.out‚ 8.41015625 disk.sda3.in‚0* proc.memory.percent‚15.569567680358887 network.recv‚161258 disk.nvme0n1.in‚0.078125) proc.memory.availableMB‚ 1962.44140625 disk.sda1.in‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 8.41015625 disk.drbd2265.in‚0& disk.nvme0c0n1.out‚712.26904296875 disk.dm-0.in‚0.078125! disk.dm-0.out‚712.26904296875 disk.sda2.out‚0 disk.rbd1.out‚ 1.142578125 disk.drbd1716.in‚0 disk.sda3.out‚ 8.41015625 disk.rbd0.in‚0# proc.memory.rssMB‚ 2550.91796875% disk./.usageGB‚296.30550384521484 disk.sda.in‚0 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 cpu‚25.19174587840108* disk./.usagePercent‚19.885310979929667 disk.sda1.out‚0$ disk.nvme0n1.out‚712.26904296875 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0¤‰e±žˆ‚(:“ µƒæÔ§Êþ gpu.0.gpu‚49.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚150.547' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚30.109399999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0|ý{©A‰‚(:¶ µƒæÔ•ÂÜ€ disk.sda1.in‚0# proc.memory.rssMB‚ 2550.91796875* proc.memory.percent‚15.569567680358887 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.rbd1.out‚ 1.205078125$ disk.nvme0n1.out‚762.56201171875& disk.nvme0c0n1.out‚762.56201171875 disk.rbd0.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125( proc.memory.availableMB‚ 1962.4453125 network.recv‚165366 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125$ memory_percent‚88.02218437194824 network.sent‚378405) disk./.usagePercent‚19.88570983966801$ disk./.usageGB‚296.3114471435547 disk.sda3.out‚ 8.89453125 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0! disk.dm-0.out‚762.56201171875 disk.sda.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda2.out‚0 disk.dm-1.out‚ 8.89453125 disk.drbd2080.in‚0 cpu‚25.18308245852345 proc.cpu.threads‚17 disk.sda.out‚ 8.894531252s‡0Š+ _wandbruntime‚226  _runtime‚2269‘ÀÄ‹¾ trainer/global_step‚299 % _timestamp‚1.7884451111713705e+09 _runtime‚ 226.643002828 _step‚20 & lr-AdamW/pg1‚3.133087025125404e-05 & lr-AdamW/pg2‚3.133087025125404e-05OzVȌ & lr-AdamW/pg1‚3.133087025125404e-05 & lr-AdamW/pg2‚3.133087025125404e-05 trainer/global_step‚299 % _timestamp‚1.7884451111713705e+09 _runtime‚ 226.643002828 _step‚20HE™Ü0+ _wandbruntime‚226  _runtime‚226+ï«'Ž¥O SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2456100 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 245664293888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚16922824140288 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚17169144029696 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚17169144029696 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚16922806137344 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚18515876610560 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚16922806137344 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚16486939595776 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3833431 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3832986 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚890 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2456102 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5569998 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚16486922556416 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 246319889408 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5551458 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1377329 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚16732353973682 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _step‚21 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 194622464 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 257839405568 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 170682368 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9782396 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚299 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚9763854 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4212398 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 246319889408 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5569998 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚17168470431232 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚18773521393664 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚871 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 257668723200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚17168470431232 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚16922824140288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 245414377906 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 244759947698 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 245664293888 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1376886 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 226.643796105 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4212396 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚16731682504114 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5551458 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚18515852670464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚17168470431232 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚18773521393664 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚871 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 257668723200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚17168470431232 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚16922824140288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 245414377906 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 244759947698 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 245664293888 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1376886 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 226.643796105 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4212396 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚16731682504114 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5551458 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚18515852670464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚16922824140288 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚17169144029696 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚17169144029696 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚16922806137344 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚18515876610560 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚16922806137344 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚16486939595776 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3833431 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3832986 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚890 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2456102 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5569998 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚16486922556416 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 246319889408 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5551458 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1377329 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚16732353973682 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _step‚21 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 194622464 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 257839405568 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 170682368 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9782396 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚299 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚9763854‰+Ÿ·0+ _wandbruntime‚227  _runtime‚2275´ž¶&‘°M OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5569912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚16979177179136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 188317696 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 247139330560 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 246483721728 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚17226316509696 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2464270 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1381465 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚16787403069638 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4226412 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9796324 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5588478 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 247139330560 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚18836031311872 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚16541843726336 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚18577503408128 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 246230813078 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚871 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚16541826686976 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3846181 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2464268 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5588478 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9814892 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚22 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 246483721728 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚890 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3845733 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 258692281344 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7884451118797157e+09 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9814892 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 245576382662 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚16979177179136 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚16979159176192 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4226412 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5569912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚16788074539414 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 164377600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚26052ýNîº&’´M D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚9796324 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5588478 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 247139330560 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚18836031311872 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚16541843726336 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚18577503408128 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 246230813078 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚871 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚16541826686976 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3846181 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2464268 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5588478 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9814892 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚22 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 246483721728 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚890 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3845733 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 258692281344 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7884451118797157e+09 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9814892 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 245576382662 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚16979177179136 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚16979159176192 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4226412 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚5569912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚16788074539414 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 164377600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5569912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚16979177179136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 188317696 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 247139330560 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 246483721728 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚17226316509696 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2464270 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1381465 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚16787403069638 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4226412 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚20027108179456 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚19740458079744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6477510 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2864502 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2864500 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚471 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚19740476082688 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 285595287106 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 287305695232 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4472069 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚21598389553664 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 @8DeviceStatsMonitor.on_train_batch_start/num_deviÈöKŸ„ce_alloc‚844 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 170682368 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚19232146123776 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 286249717314 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 286650099712 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11409248 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1607569 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚19232129084416 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 286650099712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚11390706 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 287305695232 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4913196 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚21899039115776 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 194622464 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚19740458079744 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4913198 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚25 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4913198 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚20027108179456 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4913196 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _runtime‚ 263.576331109 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚890 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚21899233738240 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚19518395841090 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚20027781777920 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 300820244480ÆHÓj´'Ÿ®O YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 194622464 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚19740458079744 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4913198 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚25 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4913198 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚20027108179456 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4913196 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _runtime‚ 263.576331109 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚890 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚21899233738240 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚19518395841090 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚20027781777920 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 300820244480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚20027108179456 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚19740458079744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6477510 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2864502 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2864500 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚471 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚19740476082688 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 285595287106 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 287305695232 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4472069 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚21598389553664 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 170682368 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚19232146123776 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 286249717314 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 286650099712 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11409248 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1607569 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚19232129084416 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 286650099712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚11390706 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 287305695232 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4913196 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚21899039115776 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0ø)/Ä0 + _wandbruntime‚264  _runtime‚264¶=§¹&¡³M MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6514530 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4484810 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚871 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚21961548542464 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6495964 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚19796811118592 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚19796829121536 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11441744 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚20084954257920 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4927212 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 286411722070 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11441744 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4927212 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6514530 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 190414848 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 288125136384 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1612142 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚19287033214976 trainer/global_step‚349 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚20084280646144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 166474752 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 301674725888 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚19573444937046 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4485275 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2872670 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2872668 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 287066152486 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚19796829121536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚21660040291328 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4927214 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚465 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚11423176 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚19287050254336 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1612605 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚463 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚19796811118592 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4927214 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605RËïC¼¢·M G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11423176 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚21660064231424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 288125136384 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚890 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚20084954257920 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 287469527552 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11441744 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4927212 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6514530 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 190414848 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 288125136384 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1612142 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚19287033214976 trainer/global_step‚349 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚20084280646144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 166474752 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 301674725888 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚19573444937046 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4485275 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2872670 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2872668 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 287066152486 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.p¶@teak‚22903 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚19796829121536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚21660040291328 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4927214 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚465 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚11423176 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚19287050254336 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1612605 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚463 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚19796811118592 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4927214 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6514530 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4484810 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚871 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚21961548542464 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6495964 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚19796811118592 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚19796829121536 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11441744 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚20084954257920 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4927212 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 286411722070 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚22558128025088 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚478 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3272900 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 328291501056 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200913920 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚22558128025088 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 327635905536 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 343827471872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚24680926436864 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 _runtime‚ 300.314502649 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚22885745927680 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚22886419526144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚22303766238930 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5613996 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 326430626514 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚7403562 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5112666 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5112188 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7422102 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚24680950376960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176973824 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚894 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7422102 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚476 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0†€t£'±®O D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5613998 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚22558110022144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13036100 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚22558128025088 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚478 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3272900 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 328291501056 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200913920 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚22558128025088 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 327635905536 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 343827471872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚24680926436864 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 _runtime‚ 300.314502649 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚22885745927680 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚22886419526144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚22303766238930 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5613996 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 326430626514 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚7403562 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5112666 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5112188 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7422102 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚24680950376960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176973824 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚894 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7422102 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚476 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 IDeviceStatsMonitor.on_train_batch_start/oversize_segmÓ2Ëents.peak‚0è6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 344686461952 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚22942918394368 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚22360158274230 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 329110942208 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 _runtime‚ 301.001046036 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 327901491894 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 $ _timestamp‚1.788445185528471e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚22614481063936 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚22032256782336 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 329110942208 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5628014 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7440582 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚399 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5628012 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 328455333376 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13068596 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚22943592006144 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚22614481063936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13068596 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚25087454051328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5124932 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3281068 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7422016 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚25087263636480 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚24742601114624 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 166474752 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚22943592006144 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1844320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚22032239742976 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚30 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚22614463060992 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 328455333376 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 327247061478 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7422016 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5125390ŠîeŽ&´·M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚22614463060992 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 328455333376 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 327247061478 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7422016 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5125390 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 344686461952 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚22942918394368 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚22360158274230 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 329110942208 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 _runtime‚ 301.001046036 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 327901491894 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 $ _timestamp‚1.788445185528471e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚22614481063936 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚22032256782336 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 329110942208 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5628014 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚7440582 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚399 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5628012 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 328455333376 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13068596 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚22943592006144 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚22614481063936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13068596 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚25087454051328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5124932 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3281068 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7422016 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚25087263636480 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚24742601114624 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 166474752 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚22943592006144 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1844320 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚22032239742976 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚30 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414’ gð0µ+ _wandbruntime‚301  _runtime‚301&mQ¶Ë 0loss/FRt1/assignment_loss‚7.3840028792619705 2loss/SRqqt3/assignment_loss‚3.3572219654365822 /loss/FBt1/assignment_loss‚0.301802139944182 0loss/SRqqt2/detection_loss‚5.435188211740986 0loss/SRqqt4/detection_loss‚3.030417130240854 .loss/FBt3/detection_loss‚4.022453159520228 1loss/SRqqt3/detection_loss‚2.2133527142629004 .loss/FBt4/detection_loss‚4.391498506938772 epoch‚0 _step‚31 1loss/FBt2/assignment_loss‚0.37019201029593557 .loss/FRt2/detection_loss‚5.082146500839906 1loss/SRqqt1/detection_loss‚1.8814988454844437 /loss/FRt3/assignment_loss‚7.082992444958603 /loss/FRt3/detection_loss‚2.5940477831060424 2loss/SRqqt2/assignment_loss‚3.4774985851780063 0loss/FBt4/assignment_loss‚0.8199196638694654 .loss/FRt1/detection_loss‚2.679202858824283 % _timestamp‚1.7884451855305402e+09 /loss/FRt2/assignment_loss‚7.169339628780589 1loss/FBt3/assignment_loss‚0.45186215276563807 %loss/total_loss‚98.84679848178047 _runtime‚ 301.002090929 .loss/FRt4/assignment_loss‚7.30360438975882 2loss/SRqqt4/assignment_loss‚3.3834408769067728 /loss/FRt4/detection_loss‚3.6348220052871296 .loss/FBt1/detection_loss‚6.406599471202264 /loss/FBt2/detection_loss‚12.550507726577612 trainer/global_step‚399 1loss/SRqqt1/assignment_loss‚3.823186830598481»F©U·Ï 0loss/FRt1/assignment_loss‚7.3840028792619705 2loss/SRqqt3/assignment_loss‚3.3572219654365822 /loss/FBt1/assignment_loss‚0.301802139944182 0loss/SRqqt2/detection_loss‚5.435188211740986 0loss/SRqqt4/detection_loss‚3.030417130240854 .loss/FBt3/detection_loss‚4.022453159520228 1loss/SRqqt3/detection_loss‚2.2133527142629004 .loss/FBt4/detection_loss‚4.391498506938772 epoch‚0 _step‚31 1loss/FBt2/assignment_loss‚0.37019201029593557 .loss/FRt2/detection_loss‚5.082146500839906 1loss/SRqqt1/detection_loss‚1.8814988454844437 /loss/FRt3/assignment_loss‚7.082992444958603 /loss/FRt3/detection_loss‚2.5940477831060424 2loss/SRqqt2/assignment_loss‚3.4774985851780063 0loss/FBt4/assignment_loss‚0.8199196638694654 .loss/FRt1/detection_loss‚2.679202858824283 % _timestamp‚1.7884451855305402e+09 /loss/FRt2/assignment_loss‚7.169339628780589 1loss/FBt3/assignment_loss‚0.45186215276563807 %loss/total_loss‚98.84679848178047 _runtime‚ 301.002090929 .loss/FRt4/assignment_loss‚7.30360438975882 2loss/SRqqt4/assignment_loss‚3.3834408769067728 /loss/FRt4/detection_loss‚3.6348220052871296 .loss/FBt1/detection_loss‚6.406599471202264 /loss/FBt2/detection_loss‚12.550507726577612 trainer/global_step‚399 1loss/SRqqt1/assignment_loss‚3.823186830598481ÎTª ž¸‚(:“ „æÔ îØþ gpu.0.gpu‚17.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚124.479' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.895799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€·¤H¹‚(:½ „æÔžÚ‹ÿ disk.nvme0c0n1.in‚0.078125* proc.memory.percent‚15.569663047790527 disk.drbd2080.out‚0 disk.drbd2266.out‚0$ memory_percent‚88.00630569458008 disk.rbd1.out‚ 1.8779296875 disk.dm-1.out‚ 12.07421875 disk.sda3.out‚ 12.07421875 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0% disk.nvme0n1.out‚1056.14697265625 network.sent‚503145 network.recv‚195777 disk.sda2.in‚0# proc.memory.rssMB‚ 2550.93359375 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.rbd0.in‚0' proc.memory.availableMB‚ 1965.046875* disk./.usagePercent‚19.890861735287064 disk.nvme0n1.in‚0.078125 cpu‚25.14780827661392 disk.dm-1.in‚0$ disk./.usageGB‚296.3882141113281' disk.nvme0c0n1.out‚1056.14697265625 disk.sda3.in‚0 disk.rbd0.out‚0" disk.dm-0.out‚1056.14697265625 disk.sda.in‚0 disk.sda.out‚ 12.07421875 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0ÉsOžº‚(:“ ž„æÔð‘Šý gpu.0.gpu‚50.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚39.0 gpu.0.powerWatts‚183.593' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.718599999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0^Å?.C»‚(:¸ ž„æÔ鸖€% disk.nvme0n1.out‚1097.22900390625 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.rbd1.out‚ 1.8779296875 disk.dm-1.out‚ 12.72265625 disk.sda3.out‚ 12.72265625 cpu‚25.14287111267773 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0) disk./.usagePercent‚19.89140626331239 disk.sda.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125$ memory_percent‚88.00802230834961* proc.memory.percent‚15.569686889648438 network.sent‚554178 network.recv‚202686$ disk./.usageGB‚296.3963279724121 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚1097.22900390625 disk.sda3.in‚0" disk.dm-0.out‚1097.22900390625 disk.sda.out‚ 12.72265625 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0' proc.memory.availableMB‚ 1964.765625 proc.memory.rssMB‚ 2550.9375 proc.cpu.threads‚17 disk.dm-1.in‚0 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0¼Vê40¼+ _wandbruntime‚336  _runtime‚33678/¨Ä½¾ trainer/global_step‚449 % _timestamp‚1.7884452212350025e+09 _runtime‚ 336.706586903 _step‚32 & lr-AdamW/pg1‚4.704869813649855e-05 & lr-AdamW/pg2‚4.704869813649855e-05 ¡TȾ & lr-AdamW/pg1‚4.704869813649855e-05 & lr-AdamW/pg2‚4.704869813649855e-05 trainer/global_step‚449 % _timestamp‚1.7884452212350025e+09 _runtime‚ 336.706586903 _step‚32 Ka0¿+ _wandbruntime‚336  _runtime‚336iÈPÀªO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚894 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 386932067328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚25090479575906 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚25375761964544 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5753282 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2071530 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 367920396130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚25744383675904 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚25375779967488 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177845248 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚875 _step‚33 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6314796 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14662952 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚27763463320064 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 367265965922 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 368621711360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚25375779967488 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 387085972480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚24722542140416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 % _timestamp‚1.7884452212357073e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5752830 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430¹Î¯Ö´'Á®O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚25090479575906 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚25375761964544 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5753282 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2071530 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 367920396130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚25744383675904 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚25375779967488 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177845248 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚875 _step‚33 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6314796 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14662952 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚27763463320064 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 367265965922 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 368621711360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚25375779967488 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 387085972480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚24722542140416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 % _timestamp‚1.7884452212357073e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5752830 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚894 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 386932067328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414!¬Óº"0Â+ _wandbruntime‚337  _runtime‚337ss͵¹&óM SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚24777446270976 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3689470 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14695448 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 369441139200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚27825114057728 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚25432115003392 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8348068 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8366634 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2076487 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 368736831302 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚25432133006336 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚875 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚28212902593024 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 387788535296 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚ 337.39929448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚27825137997824 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚8366634 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚14676880 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 369441139200 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5765498 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 % _timestamp‚1.7884452219270768e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚25432115003392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚894 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14695448 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14676880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚25801556142592 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6328814 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6328812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 368082400886 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2076030 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944™ e@à"Ä·M OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚8366634 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚14676880 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 369441139200 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5765498 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 % _timestamp‚1.7884452219270768e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚25432115003392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚894 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14695448 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚14676880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚25801556142592 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6328814 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6328812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 368082400886 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2076030 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚24777446270976 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3689470 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14695448 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 369441139200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚27825114057728 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚25432115003392 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8348068 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚8366634 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2076487 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 368736831302 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚25432133006336 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚875 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚28212902593024 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 387788535296 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚ 337.39929448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚27825137997824"Öå¤ç0Å+ _wandbruntime‚337  _runtime‚337*û‘³KÆÅ 1loss/FBt3/assignment_loss‚0.46454289140036475 % _timestamp‚1.7884452219292054e+09 trainer/global_step‚449 /loss/FRt2/assignment_loss‚7.181702351570129 1loss/SRqqt1/assignment_loss‚3.904816967248917 1loss/SRqqt2/assignment_loss‚2.404909701908336 0loss/FBt4/assignment_loss‚0.7014446411530326 .loss/FRt2/detection_loss‚3.765628957748413 %loss/total_loss‚90.79436811836078 /loss/FRt4/detection_loss‚2.6749234440426033 1loss/SRqqt1/detection_loss‚2.3216689986907584 .loss/FBt1/detection_loss‚6.529398686897296 -loss/FBt4/detection_loss‚2.04207756638781 /loss/FRt3/assignment_loss‚7.184134292602539 1loss/SRqqt4/assignment_loss‚3.304410170535652 -loss/FRt3/detection_loss‚3.07122071005404 .loss/FRt4/assignment_loss‚7.35999219417572 2loss/SRqqt3/assignment_loss‚3.2526852897569243 0loss/SRqqt3/detection_loss‚2.199174076136414 epoch‚0 /loss/FRt1/assignment_loss‚7.432113924691843 0loss/SRqqt4/detection_loss‚3.309430607119385 /loss/FBt3/detection_loss‚4.9921306687826315 .loss/FRt1/detection_loss‚4.069268551330234 0loss/SRqqt2/detection_loss‚3.259647496775085 .loss/FBt2/detection_loss‚8.455446004867554 _step‚35 1loss/FBt1/assignment_loss‚0.36335874882598335 0loss/FBt2/assignment_loss‚0.5502411756590989 _runtime‚ 337.400735058­OÇÉ epoch‚0 /loss/FRt1/assignment_loss‚7.432113924691843 0loss/SRqqt4/detection_loss‚3.309430607119385 /loss/FBt3/detection_loss‚4.9921306687826315 .loss/FRt1/detection_loss‚4.069268551330234 0loss/SRqqt2/detection_loss‚3.259647496775085 .loss/FBt2/detection_loss‚8.455446004867554 _step‚35 1loss/FBt1/assignment_loss‚0.36335874882598335 0loss/FBt2/assignment_loss‚0.5502411756590989 _runtime‚ 337.400735058 1loss/FBt3/assignment_loss‚0.46454289140036475 % _timestamp‚1.7884452219292054e+09 trainer/global_step‚449 /loss/FRt2/assignment_loss‚7.181702351570129 1loss/SRqqt1/assignment_loss‚3.904816967248917 1loss/SRqqt2/assignment_loss‚2.404909701908336 0loss/FBt4/assignment_loss‚0.7014446411530326 .loss/FRt2/detection_loss‚3.765628957748413 %loss/total_loss‚90.79436811836078 /loss/FRt4/detection_loss‚2.6749234440426033 1loss/SRqqt1/detection_loss‚2.3216689986907584 .loss/FBt1/detection_loss‚6.529398686897296 -loss/FBt4/detection_loss‚2.04207756638781 /loss/FRt3/assignment_loss‚7.184134292602539 1loss/SRqqt4/assignment_loss‚3.304410170535652 -loss/FRt3/detection_loss‚3.07122071005404 .loss/FRt4/assignment_loss‚7.35999219417572 2loss/SRqqt3/assignment_loss‚3.2526852897569243 0loss/SRqqt3/detection_loss‚2.199174076136414#A¢i“È‚(:ˆ ­„æÔ¶¾¯ý gpu.0.gpu‚49.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚175.734' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.1468 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0O0AEÉ‚(:º ­„æÔн³ÿ disk.drbd2080.in‚0% disk./.usageGB‚296.42602920532227 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.rbd1.out‚ 1.9404296875* disk./.usagePercent‚19.893399537973984 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.sda2.out‚0' disk.nvme0c0n1.out‚1150.84716796875 disk.drbd2265.out‚0 disk.sda.out‚ 13.21484375 disk.dm-1.in‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125* proc.memory.percent‚15.569686889648438 cpu‚25.139922893849594 network.sent‚559786 network.recv‚206298" disk.dm-0.out‚1150.84716796875 disk.sda2.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125# memory_percent‚88.0070686340332 proc.memory.rssMB‚ 2550.9375 disk.rbd0.in‚0' proc.memory.availableMB‚ 1964.921875 disk.sda3.in‚0 disk.sda3.out‚ 13.21484375% disk.nvme0n1.out‚1150.84716796875 disk.dm-1.out‚ 13.21484375†àeJžÊ‚(:“ ¼„æԤ͋ý gpu.0.gpu‚29.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚175.696' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.139199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Àã*GË‚(:¼ ¼„æÔ혀 network.recv‚212687 disk.drbd2265.in‚0* proc.memory.percent‚15.569686889648438 cpu‚25.135710842990626 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 proc.memory.rssMB‚ 2550.9375 proc.cpu.threads‚17$ disk./.usageGB‚296.4476852416992 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 2.0341796875' disk.nvme0c0n1.out‚1193.29248046875 disk.sda3.out‚ 13.80078125 disk.sda.out‚ 13.80078125 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 13.80078125) proc.memory.availableMB‚ 1964.85546875 network.sent‚610203 disk.drbd2266.out‚0" disk.dm-0.out‚1193.29248046875 disk.sda2.out‚0 disk.rbd1.in‚0$ memory_percent‚88.00747394561768* disk./.usagePercent‚19.894852892745867 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚1193.29248046875 disk.dm-1.in‚0W@)c0Ì+ _wandbruntime‚374  _runtime‚374†ô‡ÆÍÀ _runtime‚ 374.127024298 _step‚36 ' lr-AdamW/pg1‚5.2287974098246715e-05 ' lr-AdamW/pg2‚5.2287974098246715e-05 trainer/global_step‚499 % _timestamp‚1.7884452586553783e+09Ê9CMÊÎÄ ' lr-AdamW/pg1‚5.2287974098246715e-05 ' lr-AdamW/pg2‚5.2287974098246715e-05 trainer/global_step‚499 % _timestamp‚1.7884452586553783e+09 _runtime‚ 374.127024298 _step‚36$ø]¢Ø0Ï+ _wandbruntime‚374  _runtime‚374gx_°'ЪO ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 408755735538 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚28193413906944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 408101305330 trainer/global_step‚499 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177845248 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9274206 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 409607517184 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6392618 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9274206 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚28193431909888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚16289804 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚27467748668416 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚439 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 153905152 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7015598 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7015598 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6392179 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4089702 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚27875849973746 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 % _timestamp‚1.7884452586561103e+09 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7015596 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9255666 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚28603695022592 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16271262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚30846024143360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9255666 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚437 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 410263112704 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚31276126892032 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚28603695022592 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 409607517184 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚28603021424128 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚875 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 410263112704 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚28193431909888 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚31276304737280 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚28193413906944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2302479 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚894 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 430126688768 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚28603021424128 _runtime‚ 374.128033032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4089700 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚27467765707776 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384gµâ´'Ñ®O _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚28193431909888 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚31276304737280 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚28193413906944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2302479 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚894 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 430126688768 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚28603021424128 _runtime‚ 374.128033032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4089700 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚27467765707776 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 408755735538 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚28193413906944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 408101305330 trainer/global_step‚499 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177845248 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9274206 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 409607517184 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6392618 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9274206 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚28193431909888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚16289804 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚27467748668416 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚439 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 153905152 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7015598 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7015598 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6392179 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4089702 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚27875849973746 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 % _timestamp‚1.7884452586561103e+09 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7015596 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9255666 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚28603695022592 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16271262 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚30846024143360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9255666 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚437 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 410263112704 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚31276126892032 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚28603695022592 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 409607517184 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚28603021424128 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚875 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 410263112704%÷ƒA0Ò+ _wandbruntime‚374  _runtime‚3749Š%j¨Ó´M RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7029614 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4097868 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚27522669838336 % _timestamp‚1.7884452593544009e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 410426945024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 182026240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7029614 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚27522652798976 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4097870 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7029612 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7029612 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2307086 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceS }øtatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9292686 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚16303732 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6404954 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚499 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6405395 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30907650940928 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚28660193890816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 _runtime‚ 374.826523661 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚30907674881024 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 411082553856 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚28660867502592 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚27932242009046 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚28249784948736 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2307525 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚28249766945792 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚28249784948736 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚28249766945792 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16322300 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4097870 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7029612 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7029612 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2307086 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9292686 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚16303732 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6404954 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚499 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6405395 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30907650940928 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚28660193890816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 _runtime‚ 374.826523661 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚30907674881024 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 411082553856 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚28660867502592 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚27932242009046 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚28249784948736 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2307525 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚28249766945792 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚28249784948736 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚28249766945792 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16322300 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0&µ_Ö0Õ+ _wandbruntime‚374  _runtime‚3743ý‘QÖË 1loss/FBt2/assignment_loss‚0.42822804882656784 0loss/FBt4/assignment_loss‚0.6256875434234698 .loss/FBt1/detection_loss‚3.316987039044846 trainer/global_step‚499 2loss/SRqqt1/assignment_loss‚3.5190190009351046 /loss/FRt3/detection_loss‚3.2084091002557504 /loss/FRt2/assignment_loss‚7.230547580718994 2loss/SRqqt4/assignment_loss‚3.0392842250957823 /loss/FRt1/detection_loss‚2.6812468126416205 1loss/SRqqt1/detection_loss‚1.8561044071343171 -loss/FBt3/detection_loss‚4.06601737174767 epoch‚0 _runtime‚ 374.827694681 2loss/SRqqt3/assignment_loss‚3.2641066951411113 .loss/FRt4/assignment_loss‚7.29170090675354 0loss/FBt3/assignment_loss‚0.5705816459477596 0loss/SRqqt3/detection_loss‚2.642214308745763 1loss/SRqqt4/detection_loss‚2.6688103231421687 /loss/FRt4/detection_loss‚3.4392143791913985 1loss/SRqqt2/assignment_loss‚2.747307905129024 0loss/FBt1/assignment_loss‚0.2231640327589175 .loss/FRt2/detection_loss‚3.349103121757507 1loss/SRqqt2/detection_loss‚3.8936033092793965 .loss/FBt2/detection_loss‚6.235700761079788 % _timestamp‚1.7884452593561738e+09 _step‚39 /loss/FRt1/assignment_loss‚7.415828529993693 /loss/FBt4/detection_loss‚3.1301105115568575 %loss/total_loss‚83.98680962499051 /loss/FRt3/assignment_loss‚7.143832064689474¾$a‘U×Ï /loss/FBt4/detection_loss‚3.1301105115568575 %loss/total_loss‚83.98680962499051 /loss/FRt3/assignment_loss‚7.143832064689474 1loss/FBt2/assignment_loss‚0.42822804882656784 0loss/FBt4/assignment_loss‚0.6256875434234698 .loss/FBt1/detection_loss‚3.316987039044846 trainer/global_step‚499 2loss/SRqqt1/assignment_loss‚3.5190190009351046 /loss/FRt3/detection_loss‚3.2084091002557504 /loss/FRt2/assignment_loss‚7.230547580718994 2loss/SRqqt4/assignment_loss‚3.0392842250957823 /loss/FRt1/detection_loss‚2.6812468126416205 1loss/SRqqt1/detection_loss‚1.8561044071343171 -loss/FBt3/detection_loss‚4.06601737174767 epoch‚0 _runtime‚ 374.827694681 2loss/SRqqt3/assignment_loss‚3.2641066951411113 .loss/FRt4/assignment_loss‚7.29170090675354 0loss/FBt3/assignment_loss‚0.5705816459477596 0loss/SRqqt3/detection_loss‚2.642214308745763 1loss/SRqqt4/detection_loss‚2.6688103231421687 /loss/FRt4/detection_loss‚3.4392143791913985 1loss/SRqqt2/assignment_loss‚2.747307905129024 0loss/FBt1/assignment_loss‚0.2231640327589175 .loss/FRt2/detection_loss‚3.349103121757507 1loss/SRqqt2/detection_loss‚3.8936033092793965 .loss/FBt2/detection_loss‚6.235700761079788 % _timestamp‚1.7884452593561738e+09 _step‚39 /loss/FRt1/assignment_loss‚7.415828529993693'l71“Ø‚(:ˆ Ë„æÔýøý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.503' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.9006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0éxFHÙ‚(:½ Ë„æÔÛÅ€ disk.rbd1.out‚ 2.2216796875 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 14.19140625 proc.cpu.threads‚17 disk.dm-0.in‚0.078125 disk.dm-1.in‚0* proc.memory.percent‚15.569686889648438 disk.dm-1.out‚ 14.19140625 network.recv‚216324$ memory_percent‚88.00899982452393' disk.nvme0c0n1.out‚1240.54345703125 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0) proc.memory.availableMB‚ 1964.60546875 proc.memory.rssMB‚ 2550.9375 disk.drbd2266.in‚0 disk.sda3.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚1240.54345703125 cpu‚25.132538751146498 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 network.sent‚615691* disk./.usagePercent‚19.895210023256134% disk./.usageGB‚296.45300674438477" disk.dm-0.out‚1240.54345703125 disk.sda.in‚0 disk.sda.out‚ 14.19140625 disk.sda1.out‚0 disk.rbd0.out‚0#T8ÄžÚ‚(:“ Ú„æÔÍ̈ý gpu.0.gpu‚45.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.937' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.187400000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0åŸGÛ‚(:¼ Ú„æÔù“ÿ disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚1279.45751953125 disk.sda3.in‚0% disk.nvme0n1.out‚1279.45751953125 proc.memory.rssMB‚ 2550.9375 proc.cpu.threads‚17 disk.sda3.out‚ 14.98046875 disk.drbd2266.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚0( proc.memory.availableMB‚ 1964.5078125 network.recv‚223111 disk.rbd0.out‚0" disk.dm-0.out‚1279.45751953125 disk.dm-1.out‚ 14.98046875$ memory_percent‚88.00959587097168* disk./.usagePercent‚19.895516976289688 disk.drbd2266.in‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 network.sent‚666309 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2265.in‚0% disk./.usageGB‚296.45758056640625 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.rbd1.out‚ 2.2216796875 disk.drbd2265.out‚0 cpu‚25.129705196718987 disk.sda.out‚ 14.98046875 disk.sda1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0* proc.memory.percent‚15.569686889648438)Œcª“Ü‚(:ˆ é„æÔÀþõü gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.271' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.4542 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ŸnõAÝ‚(:¶ é„æÔÿÚîþ( disk./.usagePercent‚19.8961129618294$ disk./.usageGB‚296.4664611816406 disk.dm-0.in‚0.078125 disk.sda2.in‚0 network.sent‚671870 disk.sda.out‚ 15.45703125 disk.rbd1.in‚0# disk.nvme0n1.out‚1315.357421875 disk.drbd2265.out‚0* proc.memory.percent‚15.569710731506348 disk.rbd0.out‚0 disk.sda2.out‚0 disk.drbd1716.out‚0' proc.memory.availableMB‚ 1964.734375 disk.dm-1.out‚ 15.45703125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 cpu‚25.126986794766154 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.dm-0.out‚1315.357421875# proc.memory.rssMB‚ 2550.94140625 network.recv‚226940% disk.nvme0c0n1.out‚1315.357421875 disk.sda3.in‚0 disk.sda1.in‚0 disk.rbd1.out‚ 2.2841796875 disk.drbd2080.in‚0 disk.sda3.out‚ 15.45703125 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd1716.in‚0$ memory_percent‚88.008213043212893,Ï`0Þ+ _wandbruntime‚411  _runtime‚411.ŽÈÃß½ $ _timestamp‚1.788445296514384e+09 _runtime‚ 411.986054696 _step‚40 & lr-AdamW/pg1‚5.752725005999489e-05 & lr-AdamW/pg2‚5.752725005999489e-05 trainer/global_step‚549|hÜ'ÇàÁ & lr-AdamW/pg2‚5.752725005999489e-05 trainer/global_step‚549 $ _timestamp‚1.788445296514384e+09 _runtime‚ 411.986054696 _step‚40 & lr-AdamW/pg1‚5.752725005999489e-05(ܶ•;0á+ _wandbruntime‚411  _runtime‚411.îÛl'â®O HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚31462332770816 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚31011065849344 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10181718 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚31011065849344 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 450593323008 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4498102 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚ 411.986927974 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚34402037870592 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚30212972235776 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 _step‚41 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚31461659172352 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7032926 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 153905152 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7716398 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 trainer/global_step‚549 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10200258 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 450593323008 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚31461659172352 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚33928537086464 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 473476844032 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚31011083852288 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7716396 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7716398 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚31011083852288 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚33928561026560 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚30212955196416 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7032461 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚30662563310722 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚465 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7716396 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 451248918528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177845248 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 473322938880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10200258 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚894 SGDeviceStaž*ÎFHtsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2534361¯k¾¸'ã²O OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚31011083852288 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7716396 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7716398 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚31011083852288 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚33928561026560 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚30212955196416 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7032461 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚30662563310722 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚465 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7716396 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 451248918528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177845248 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 473322938880 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10200258 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚894 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2534361 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚31462332770816 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚31011065849344 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10181718 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚31011065849344 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 450593323008 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4498102 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚ 411.986927974 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚34402037870592 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚30212972235776 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 _step‚41 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚31461659172352 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7032926 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 153905152 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7716398 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 trainer/global_step‚549 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10200258 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 450593323008 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚31461659172352 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚33928537086464 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 473476844032 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844)A&0ä+ _wandbruntime‚412  _runtime‚412MÔ½&å·M QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2538926 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚30267876366336 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚34464384781824 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚ 412.696324009 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7730412 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 449753079702 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 451412750848 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚30717612406678 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚31067418888192 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 452068359680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚31067436891136 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 $ _timestamp‚1.788445297224267e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 trainer/global_step‚549 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7730414 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10218738 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2539389 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7730414 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10218738 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚875 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17949152 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚31519505250816 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _step‚42 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177831936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 474350849536 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17949152 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 450407510118 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚894 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10200172 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4506270 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 474196957696 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7045194 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚31519505250816 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7730412 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153891840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚31067436891136ºøÁ&æ»M UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4506270 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 474196957696 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7045194 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚31519505250816 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7730412 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153891840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚31067436891136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2538926 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚30267876366336 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚34464384781824 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _runtime‚ 412.696324009 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7730412 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 449753079702 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 451412750848 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚30717612406678 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚31067418888192 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 452068359680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚31067436891136 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 $ _timestamp‚1.788445297224267e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 trainer/global_step‚549 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7730414 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚10218738 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2539389 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7730414 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10218738 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚875 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17949152 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚31519505250816 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _step‚42 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177831936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 474350849536 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17949152 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 450407510118 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚894 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10200172 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0*Ð\­­0ç+ _wandbruntime‚412  _runtime‚412‘©QèË /loss/FRt3/assignment_loss‚7.170601031359504 2loss/SRqqt4/assignment_loss‚2.7446918189525604 0loss/SRqqt2/detection_loss‚4.607193431920475 .loss/FBt4/detection_loss‚4.738179804194595 %loss/total_loss‚83.81580957486776 epoch‚0 /loss/FRt4/assignment_loss‚7.354980693673188 2loss/SRqqt3/assignment_loss‚3.1875605840940735 /loss/FRt2/assignment_loss‚7.236278305882993 .loss/FBt1/detection_loss‚2.584831616753396 /loss/FBt3/detection_loss‚1.5091674116334441 0loss/FBt1/assignment_loss‚0.3258422794372924 /loss/FBt4/assignment_loss‚0.574381063763912 1loss/SRqqt3/detection_loss‚3.8947127407556805 _runtime‚ 412.697708943 _step‚43 2loss/SRqqt2/assignment_loss‚2.4010018524196415 /loss/FBt2/assignment_loss‚0.726200473397815 0loss/FBt3/assignment_loss‚0.6896800976091375 .loss/FBt2/detection_loss‚6.238852464283506 trainer/global_step‚549 2loss/SRqqt1/assignment_loss‚3.1117110681533813 /loss/FRt1/detection_loss‚3.5722446891855686 .loss/FRt3/detection_loss‚2.755993341954023 1loss/SRqqt1/detection_loss‚2.1587335115417168 1loss/SRqqt4/detection_loss‚2.2921086685224012 /loss/FRt1/assignment_loss‚7.386581887590125 % _timestamp‚1.7884452972261891e+09 /loss/FRt2/detection_loss‚3.4269924993100376 /loss/FRt4/detection_loss‚3.1272882384792813]~ ø™éÏ 2loss/SRqqt3/assignment_loss‚3.1875605840940735 /loss/FRt2/assignment_loss‚7.236278305882993 .loss/FBt1/detection_loss‚2.584831616753396 /loss/FBt3/detection_loss‚1.5091674116334441 0loss/FBt1/assignment_loss‚0.3258422794372924 /loss/FBt4/assignment_loss‚0.574381063763912 1loss/SRqqt3/detection_loss‚3.8947127407556805 _runtime‚ 412.697708943 _step‚43 2loss/SRqqt2/assignment_loss‚2.4010018524196415 /loss/FBt2/assignment_loss‚0.726200473397815 0loss/FBt3/assignment_loss‚0.6896800976091375 .loss/FBt2/detection_loss‚6.238852464283506 trainer/global_step‚549 2loss/SRqqt1/assignment_loss‚3.1117110681533813 /loss/FRt1/detection_loss‚3.5722446891855686 .loss/FRt3/detection_loss‚2.755993341954023 1loss/SRqqt1/detection_loss‚2.1587335115417168 1loss/SRqqt4/detection_loss‚2.2921086685224012 /loss/FRt1/assignment_loss‚7.386581887590125 % _timestamp‚1.7884452972261891e+09 /loss/FRt2/detection_loss‚3.4269924993100376 /loss/FRt4/detection_loss‚3.1272882384792813 /loss/FRt3/assignment_loss‚7.170601031359504 2loss/SRqqt4/assignment_loss‚2.7446918189525604 0loss/SRqqt2æÒl¼/detection_loss‚4.607193431920475 .loss/FBt4/detection_loss‚4.738179804194595 %loss/total_loss‚83.81580957486776 epoch‚0 /loss/FRt4/assignment_loss‚7.354980693673188+-à4q“ê‚(:ˆ ø„æÔ­§íü gpu.0.gpu‚45.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚164.628' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚32.9256 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0÷e·kCë‚(:¸ ø„æÔ¹ßüþ disk.sda2.in‚0 disk.drbd2265.out‚0) proc.memory.availableMB‚ 1964.37890625 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.rbd1.out‚ 2.3701171875 disk.nvme0n1.in‚0.078125# disk.nvme0n1.out‚1350.568359375 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0* proc.memory.percent‚15.569758415222168 disk.rbd0.out‚0 disk.sda2.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125$ memory_percent‚88.01038265228271 disk.sda.in‚0 disk.sda.out‚ 15.86328125 disk.dm-1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17( disk./.usagePercent‚19.8968845685376 disk.dm-0.in‚0.078125 disk.dm-0.out‚1350.568359375 disk.sda1.in‚0 disk.dm-1.out‚ 15.86328125# proc.memory.rssMB‚ 2550.94921875 cpu‚25.124404012608906 network.sent‚678661 disk.sda3.in‚0 disk.sda3.out‚ 15.86328125 disk.drbd2266.out‚0 network.recv‚232100$ disk./.usageGB‚296.4779586791992% disk.nvme0c0n1.out‚1350.568359375zy:“ì‚(:ˆ ‡…æÔεšý gpu.0.gpu‚36.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚126.224' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.2448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0{vZGí‚(:¼ ‡…æÔ—Ÿºÿ disk.dm-1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda1.in‚0$ disk./.usageGB‚296.4610900878906 disk.rbd1.out‚ 2.5576171875# proc.memory.rssMB‚ 2550.94921875 disk.sda2.out‚0 disk.drbd2080.in‚0 disk.sda1.out‚0 network.recv‚236725 disk.dm-1.out‚ 16.52734375 disk.sda3.out‚ 16.52734375 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 16.52734375* proc.memory.percent‚15.569758415222168 proc.cpu.threads‚17 disk.drbd1716.in‚0 disk.sda.in‚0 network.sent‚727783) disk./.usagePercent‚19.89575250322119 disk.drbd2265.in‚0 disk.sda3.in‚0" disk.dm-0.out‚1393.66259765625 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚1393.66259765625 disk.drbd1716.out‚0 disk.drbd2080.out‚0# memory_percent‚88.0089282989502 disk.rbd1.in‚0 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚1393.66259765625 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125( proc.memory.availableMB‚ 1964.6171875 cpu‚25.12194548438791ƒ”B£“î‚(:ˆ –…æÔÀ¥ˆý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.806' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.7612 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€ød?Lï‚(:Á –…æÔ›¢€* proc.memory.percent‚15.569758415222168 disk.rbd1.in‚0 disk.rbd1.out‚ 2.5576171875* disk./.usagePercent‚19.897709424812923" disk.dm-0.out‚1428.72900390625 disk.sda1.out‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.rbd0.in‚0$ memory_percent‚88.01038265228271 cpu‚25.119605072766383% disk.nvme0n1.out‚1428.72900390625 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚1428.72900390625 disk.sda3.out‚ 17.16015625) proc.memory.availableMB‚ 1964.37890625# proc.memory.rssMB‚ 2550.94921875 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0% disk./.usageGB‚296.49024963378906 disk.sda1.in‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.dm-1.out‚ 17.16015625 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 proc.cpu.threads‚17 network.sent‚734865 network.recv‚242392 disk.sda.out‚ 17.16015625 disk.sda2.out‚0 disk.sda3.in‚0 disk.sda.in‚0?Š­ù0ð+ _wandbruntime‚450  _runtime‚450\O†Äñ¾ & lr-AdamW/pg1‚6.276652602174305e-05 & lr-AdamW/pg2‚6.276652602174305e-05 trainer/global_step‚599 % _timestamp‚1.7884453346664238e+09 _runtime‚ 450.138086373 _step‚44œâ fÈò & lr-AdamW/pg1‚6.276652602174305e-05 & lr-AdamW/pg2‚6.276652602174305e-05 trainer/global_step‚599 % _timestamp‚1.7884453346664238e+09 _runtime‚ 450.138086373 _step‚44,—&B0ó+ _wandbruntime‚450  _runtime‚450Bp ý´'ô®O YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 491579128832 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8417198 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7673003 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚33828717791744 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚33828735794688 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2766503 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 492234724352 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4906500 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚33828717791744 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 490426414354 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚32958178763776 trainer/global_step‚599 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 489771984146 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 491579128832 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 516961018880 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚33828735794688 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚37528058928640 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚32958161724416 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚34320970519040 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11126310 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11107770 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37011097909760 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7884453346671095e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8417198 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8417196 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 145516544 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8417196 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4906502 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 492234724352 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2766960 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚37527889472000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚34320296920576 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 516815502336 _runtime‚ 450.138916219;Ïؾ¸'õ²O ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 490426414354 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚32958178763776 trainer/global_step‚599 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 489771984146 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 491579128832 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 516961018880 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚33828735794688 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚37528058928640 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚32958161724416 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚34320970519040 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11126310 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11107770 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37011097909760 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7884453346671095e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8417198 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8417196 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 145516544 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8417196 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4906502 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 492234724352 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2766960 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚37527889472000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚34320296920576 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 516815502336 _runtime‚ 450.138916219 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 491579128832 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8417198 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7673003 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚33828717791744 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚33828735794688 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2766503 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 492234724352 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4906500 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚33828717791744- ·ÊU0ö+ _wandbruntime‚450  _runtime‚450ûAä±Ä÷¸M C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚34378142999040 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 493054165504 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4914670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚33503654274086 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 517853663744 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚474 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8431214 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 492398556672 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚37072724707328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚34377469387264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2771577 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚33504325743862 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 491242849526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚33885070830592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚33013082894336 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19576004 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11144790 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8431214 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚19557436 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚33013065854976 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚37590602311168 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚37072748647424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 151794688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚19557436 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚33885088833536 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 493054165504 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚46 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8431212 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚472 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7685773 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FåR•ú 6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _runtime‚ 450.841598469 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚33885070830592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 490588419110 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11144790 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚33885088833536 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4914668 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19576004 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2771105 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844mèŽ&Â&ø¼M J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 trainer/global_step‚599 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 492398556672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7685773 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _runtime‚ 450.841598469 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚33885070830592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 490588419110 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11144790 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚33885088833536 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4914668 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19576004 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2771105 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚34378142999040 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 493054165504 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4914670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚33503654274086 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 517853663744 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚474 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8431214 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 492398556672 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚37072724707328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚34377469387264 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2771577 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚33504325743862 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 491242849526 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚33885070830592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚33013082894336 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚19576004 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11144790 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8431214 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚19557436 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚33013065854976 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚37590602311168 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚37072748647424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 151794688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚19557436 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚33885088833536 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 493054165504 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚46 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8431212 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚472 Bý‚(:³ ¥…æÔÇ¢†€ disk.dm-0.out‚ 1470.61328125 disk.drbd1716.out‚0 disk.sda2.out‚0 disk.drbd1716.in‚0* proc.memory.percent‚15.569758415222168 cpu‚25.11751566118159 disk.nvme0n1.in‚0.078125$ memory_percent‚88.01078796386719% proc.memory.availableMB‚ 1964.3125 disk.dm-1.out‚ 17.59765625 disk.rbd1.in‚0 disk.rbd1.out‚ 2.6201171875* disk./.usagePercent‚19.898456454797834 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0# proc.memory.rssMB‚ 2550.94921875 network.sent‚740425 disk.drbd2080.in‚0$ disk.nvme0c0n1.out‚ 1470.61328125 disk.sda3.in‚0 proc.cpu.threads‚17 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.rbd0.in‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.sda.in‚0 disk.sda1.out‚0 network.recv‚245990% disk./.usageGB‚296.50138092041016 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 17.59765625 disk.sda1.in‚0" disk.nvme0n1.out‚ 1470.61328125 disk.dm-1.in‚0 disk.sda3.out‚ 17.59765625 disk.drbd2266.in‚0Ö_kú“þ‚(:ˆ ´…æÔ©Ÿ•ý gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.082' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2164 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0@ê=uAÿ‚(:¶ ´…æԻ͒ÿ disk.rbd1.in‚0 disk.dm-1.in‚0) proc.memory.availableMB‚ 1964.55078125 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.out‚ 1504.95703125" disk.nvme0n1.out‚ 1504.95703125 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda.in‚0 disk.sda2.in‚0* proc.memory.percent‚15.569758415222168 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 18.14453125 disk.rbd0.out‚0 disk.drbd1716.in‚0$ memory_percent‚88.00933361053467 disk.sda.out‚ 18.14453125 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.drbd2265.in‚0 network.sent‚789982% disk./.usageGB‚296.50873947143555 disk.sda1.in‚0 disk.rbd1.out‚ 2.6201171875# proc.memory.rssMB‚ 2550.94921875 cpu‚25.1156189105562 proc.cpu.threads‚17 network.recv‚251737 disk.sda3.in‚0 disk.dm-1.out‚ 18.14453125 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125* disk./.usagePercent‚19.898950293331385$ disk.nvme0c0n1.out‚ 1504.95703125 disk.drbd2266.out‚0 disk.sda1.out‚0äwým0€+ _wandbruntime‚487  _runtime‚487Âë@Áý & lr-AdamW/pg1‚6.800580198349121e-05 & lr-AdamW/pg2‚6.800580198349121e-05 trainer/global_step‚649 $ _timestamp‚1.788445372342859e+09 _runtime‚ 487.814462372 _step‚48_J™Ç‚Á & lr-AdamW/pg1‚6.800580198349121e-05 & lr-AdamW/pg2‚6.800580198349121e-05 trainer/global_step‚649 $ _timestamp‚1.788445372342859e+09 _runtime‚ 487.814462372 _step‚480KjvÖ0ƒ+ _wandbruntime‚487  _runtime‚487äÅ«³'„­O UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 532564934656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚37179608267264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12052362 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2997161 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9117998 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9117996 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚36646387737088 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9117996 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚36233975575970 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚35703385291776 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 530607323554 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚649 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚40093610852864 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12033822 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚36646369734144 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8312520 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚49 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8312061 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12052362 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 531261753762 % _timestamp‚1.7884453723437338e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 533220530176 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12033822 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚40653730135040 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 190428160 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚36646387737088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5314902 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚40093634792960 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5314900 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚35703368252416 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚36646369734144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚40653539706880 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚37178934668800 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 532564934656 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚37179608267264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 533220530176 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 487.81570045 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 559928854016 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21170360 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 560095342080 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚40653730135040 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 190428160 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚36646387737088 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5314902 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚40093634792960 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5314900 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚35703368252416 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 A7DeviceStatsMonitor.on_train_batch_start/activwk!†èe.all.peak‚22903 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚36646369734144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚40653539706880 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚37178934668800 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 532564934656 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚37179608267264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 533220530176 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚ 487.81570045 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 559928854016 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21170360 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 560095342080 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚37236107135488 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 532078188934 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 $ _timestamp‚1.788445373080093e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 560811971584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚50 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚37236780747264 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚36702722772992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚35758289422336 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177831936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚37236780747264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3001789 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 533384362496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5323068 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3002237 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚21184288 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 trainer/global_step‚649 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 534039971328 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12052276 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5323070 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21202856 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9132012 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9132012 _runtime‚ 488.55216138 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12070842 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚36702740775936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12070842 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚40155261590528 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚36290367611270 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚40155285530624 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21202856 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 534039971328 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚40716073562112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153891840 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8324857 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 560965863424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚36289696141494 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 531423758518 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚40716251394048 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12052276 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928ÑïÄ%À&ˆºM I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21202856 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9132012 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9132012 _runtime‚ 488.55216138 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚12070842 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚36702740775936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12070842 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚40155261590528 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚36290367611270 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚40155285530624 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚21202856 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 534039971328 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚40716073562112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153891840 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8324857 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 560965863424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚36289696141494 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 531423758518 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚40716251394048 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12052276 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8325307 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9132014 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9132014 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚37236107135488 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 532078188934 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 $ _timestamp‚1.788445373080093e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 560811971584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚50 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚37236780747264 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚36702722772992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚35758289422336 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177831936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚37236780747264 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3001789 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 533384362496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5323068 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3002237 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚21184288 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 trainer/global_step‚649 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 534039971328 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12052276 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5323070 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 8682209282t Ž*0‰+ _wandbruntime‚488  _runtime‚488_½3UOŠÉ .loss/FBt2/detection_loss‚4.595751685090363 _runtime‚ 488.553332401 2loss/SRqqt3/assignment_loss‚2.9082963241905464 .loss/FRt4/detection_loss‚2.825956614132513 0loss/FBt3/assignment_loss‚0.5847178162135499 .loss/FBt1/detection_loss‚3.282160527755095 _step‚51 /loss/FRt4/assignment_loss‚7.323597506472939 trainer/global_step‚649 .loss/FRt2/detection_loss‚4.453378679959671 .loss/FRt3/detection_loss‚2.532828258882676 0loss/SRqqt3/detection_loss‚2.082011853622287 1loss/SRqqt4/detection_loss‚1.7882123527602412 /loss/FBt3/detection_loss‚1.7769400046069899 %loss/total_loss‚85.28583894730001 /loss/FRt1/assignment_loss‚7.361263185739517 /loss/FRt2/assignment_loss‚7.275624461795973 2loss/SRqqt4/assignment_loss‚2.9124750133971093 1loss/SRqqt1/detection_loss‚3.4277816609507554 .loss/FBt4/detection_loss‚7.951037930117713 % _timestamp‚1.7884453730818186e+09 0loss/SRqqt1/assignment_loss‚3.43495003638729 1loss/SRqqt2/assignment_loss‚2.325635416640176 0loss/FBt2/assignment_loss‚0.6010669747192878 0loss/SRqqt2/detection_loss‚4.518997559944789 /loss/FRt1/detection_loss‚3.2864095078160367 0loss/FRt3/assignment_loss‚7.2504440035138815 1loss/FBt1/assignment_loss‚0.32306126881235814 0loss/FBt4/assignment_loss‚0.4632403037782448 epoch‚0¼ncfS‹Í epoch‚0 .loss/FBt2/detection_loss‚4.595751685090363 _runtime‚ 488.553332401 2loss/SRqqt3/assignment_loss‚2.9082963241905464 .loss/FRt4/detection_loss‚2.825956614132513 0loss/FBt3/assignment_loss‚0.5847178162135499 .loss/FBt1/detection_loss‚3.282160527755095 _step‚51 /loss/FRt4/assignment_loss‚7.323597506472939 trainer/global_step‚649 .loss/FRt2/detection_loss‚4.453378679959671 .loss/FRt3/detection_loss‚2.532828258882676 0loss/SRqqt3/detection_loss‚2.082011853622287 1loss/SRqqt4/detection_loss‚1.7882123527602412 /loss/FBt3/detection_loss‚1.7769400046069899 %loss/total_loss‚85.28583894730001 /loss/FRt1/assignment_loss‚7.361263185739517 /loss/FRt2/assignment_loss‚7.275624461795973 2loss/SRqqt4/assignment_loss‚2.9124750133971093 1loss/SRqqt1/detection_loss‚3.4277816609507554 .loss/FBt4/detection_loss‚7.951037930117713 % _timestamp‚1.7884453730818186e+09 0loss/SRqqt1/assignment_loss‚3.43495003638729 1loss/SRqqt2/assignment_loss‚2.325635416640176 0loss/FBt2/assignment_loss‚0.6010669747192878 0loss/SRqqt2/detection_loss‚4.518997559944789 /loss/FRt1/detection_loss‚3.2864095078160367 0loss/FRt3/assignment_loss‚7.2504440035138815 1loss/FBt1/assignment_loss‚0.32306126881235814 0loss/FBt4/assignment_loss‚0.463240303778244832†!Œ‚(:’ Ã…æÔé½ùü gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚179.193' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚35.83860000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0+Õåo:‚(:¯ Ã…æÔ´ëÿÿ disk.dm-1.out‚18.625 disk.sda3.in‚0 disk.dm-0.in‚0.078125% disk.nvme0n1.out‚1559.77294921875$ disk./.usageGB‚296.5320854187012 disk.sda2.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125* proc.memory.percent‚15.569758415222168" disk.dm-0.out‚1559.77294921875 disk.rbd1.out‚ 2.77734375 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0$ memory_percent‚88.00723552703857# proc.memory.rssMB‚ 2550.94921875 proc.cpu.threads‚17 disk.sda3.out‚18.625 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0' disk.nvme0c0n1.out‚1559.77294921875) proc.memory.availableMB‚ 1964.89453125* disk./.usagePercent‚19.900517059440936 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 network.sent‚795563 disk.drbd2266.out‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 cpu‚25.113657183916025 network.recv‚255459 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda.out‚18.625ηœažŽ‚(:“ Ò…æÔü£–ý gpu.0.gpu‚33.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚127.736' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.547200000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¬‚ÔªD‚(:¹ Ò…æÔ²ù©€$ disk./.usageGB‚296.5540580749512 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 2.77734375 cpu‚25.11185421153852 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚1598.67138671875 disk.dm-1.in‚0 disk.sda.out‚ 19.1953125 disk.rbd1.in‚0* proc.memory.percent‚15.569758415222168 network.sent‚846301 network.recv‚262083 disk.sda3.out‚ 19.1953125 disk.drbd2266.out‚0 disk.rbd0.in‚0 proc.cpu.threads‚17 disk.dm-0.in‚0.078125 disk.sda2.out‚0) disk./.usagePercent‚19.90199166283816 disk.drbd1716.out‚0 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚1598.67138671875 disk.sda3.in‚0 disk.drbd2266.in‚0" disk.dm-0.out‚1598.67138671875 disk.dm-1.out‚ 19.1953125) proc.memory.availableMB‚ 1964.86328125 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.sda1.out‚0$ memory_percent‚88.00742626190186# proc.memory.rssMB‚ 2550.94921875 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0;ÃÊ‚(:ˆ á…æÔ«ªý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚174.136' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.8272 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErroÒØëèrs‚0 /)úA‘‚(:¶ á…æԡшÿ disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd1716.out‚0$ disk./.usageGB‚296.5615463256836 disk.sda2.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 2.83984375 disk.nvme0n1.in‚0.078125 network.recv‚265829 disk.sda2.out‚0 disk.sda3.in‚0 disk.drbd2266.in‚0' proc.memory.availableMB‚ 1964.859375 cpu‚25.109846910445636 network.sent‚851789 disk.dm-1.in‚0 disk.sda1.out‚0$ memory_percent‚88.00745010375977 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚1637.658203125 disk.rbd0.in‚0 disk.dm-0.out‚1637.658203125 disk.dm-1.out‚ 19.79296875* disk./.usagePercent‚19.902494205627875 disk.sda.out‚ 19.79296875 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125# proc.memory.rssMB‚ 2550.94921875 disk.sda3.out‚ 19.79296875 disk.sda1.in‚0* proc.memory.percent‚15.569758415222168# disk.nvme0n1.out‚1637.658203125 disk.drbd1716.in‚0 disk.drbd2080.in‚0ûr70’+ _wandbruntime‚525  _runtime‚525D‹+’Á“» $ _timestamp‚1.788445409847451e+09 _runtime‚ 525.319085713 _step‚52 % lr-AdamW/pg1‚7.32450779452394e-05 % lr-AdamW/pg2‚7.32450779452394e-05 trainer/global_step‚699â/ºÆÅ”¿ $ _timestamp‚1.788445409847451e+09 _runtime‚ 525.319085713 _step‚52 % lr-AdamW/pg1‚7.32450779452394e-05 % lr-AdamW/pg2‚7.32450779452394e-05 trainer/global_step‚69948lò0•+ _wandbruntime‚525  _runtime‚525þq´'–®O OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9818798 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179942400 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 574206336000 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚43176147736064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚40038246015488 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚40037572417024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 574206336000 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚40037572417024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚43779440921088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚40038246015488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚39020688912946 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 572097093170 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22797212 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚463 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚43779620863488 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 trainer/global_step‚699 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚39464039679488 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3230334 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12978414 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5723302 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚38448574780416 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚39464021676544 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9818796 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9818796 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884454098482187e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8953634 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3230797“ød¸'—²O ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚43779620863488 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 trainer/global_step‚699 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚39464039679488 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3230334 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12978414 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5723302 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚38448574780416 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚39464021676544 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9818796 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9818796 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884454098482187e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8953634 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3230797 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9818798 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179942400 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 574206336000 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚43176147736064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚40038246015488 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚40037572417024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 574206336000 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚40037572417024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚43779440921088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚40038246015488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚39020688912946 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 572097093170 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22797212 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚4635ˆ˜õ"0˜+ _wandbruntime‚526  _runtime‚526ÑÕÆŸ¾&™¸M SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 574370168320 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12996894 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5731470 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12978328 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚475 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 166474752 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9832814 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚39075738008902 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 572259097926 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚39520392718336 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚ 526.013191136 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 575025777152 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8966350 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43237822413824 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3235357 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚39076409478678 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 trainer/global_step‚699 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22811140 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22811140 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚39520374715392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚39520374715392 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 572913528342 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9832812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚40095418495488 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43237798473728 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚38503495950336 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 574370168320 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 604150903296 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12978328 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5731468 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 190414848 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 575025777152 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 604317378048 % _timestamp‚1.7884454105410056e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22829708 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9832814 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22829708 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8966827 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚40095418495488 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚40094744883712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚477 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3234882 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚43842139791872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚39520392718336ò HÍ,š¼M WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 572259097926 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚39520392718336 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚ 526.013191136 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 575025777152 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8966350 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43237822413824 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_endm1»Ö–#/allocation.all.peak‚22903 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3235357 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚39076409478678 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 trainer/global_step‚699 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22811140 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22811140 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚39520374715392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚39520374715392 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 572913528342 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9832812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚40095418495488 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43237798473728 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚38503495950336 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 574370168320 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 604150903296 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚12978328 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5731468 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 190414848 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 575025777152 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 604317378048 % _timestamp‚1.7884454105410056e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22829708 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9832814 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22829708 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8966827 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚40095418495488 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚40094744883712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚477 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3234882 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚43842139791872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚39520392718336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 574370168320 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12996894 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5731470 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12978328 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚475 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 166474752 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9832814 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚39075738008902 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚26'…~™0›+ _wandbruntime‚526  _runtime‚526¨CÿŒOœÉ 0loss/FBt3/assignment_loss‚0.4305479713964949 .loss/FRt2/detection_loss‚3.538581973976559 0loss/SRqqt2/detection_loss‚4.272438008534281 %loss/total_loss‚87.45297973265264 0loss/FBt1/assignment_loss‚0.2726037961671109 /loss/FRt1/detection_loss‚3.7523721080450785 % _timestamp‚1.7884454105427327e+09 _runtime‚ 526.014234356 1loss/SRqqt3/assignment_loss‚2.707485795021057 1loss/SRqqt4/assignment_loss‚2.760960892719381 .loss/FBt4/detection_loss‚8.644195234775543 /loss/FRt4/assignment_loss‚7.354853831397163 1loss/SRqqt2/assignment_loss‚2.847109004070884 1loss/FBt2/assignment_loss‚0.24567615071508295 .loss/FRt4/detection_loss‚3.604086081849204 0loss/SRqqt3/detection_loss‚2.129034755455262 .loss/FBt2/detection_loss‚5.053790434681136 /loss/FRt2/assignment_loss‚7.131500482559204 0loss/FBt4/assignment_loss‚0.8752146825194359 /loss/FRt3/detection_loss‚2.7707434328589238 0loss/SRqqt1/detection_loss‚2.100429655901588 /loss/FBt1/detection_loss‚2.9309210046974723 .loss/FBt3/detection_loss‚3.765505507514805 _step‚55 0loss/FRt1/assignment_loss‚7.3923837911515005 /loss/FRt3/assignment_loss‚7.221761712488139 2loss/SRqqt1/assignment_loss‚3.5917076693640815 1loss/SRqqt4/detection_loss‚2.0590757547932514 epoch‚0 trainer/global_step‚699¥¦N(SÍ 1loss/SRqqt2/assignment_loss‚2.847109004070884 1loss/FBt2/assignment_loss‚0.24567615071508295 .loss/FRt4/detection_loss‚3.604086081849204 0loss/SRqqt3/detection_loss‚2.129034755455262 .loss/FBt2/detection_loss‚5.053790434681136 /loss/FRt2/assignment_loss‚7.131500482559204 0loss/FBt4/assignment_loss‚0.8752146825194359 /loss/FRt3/detection_loss‚2.7707434328589238 0loss/SRqqt1/detection_loss‚2.100429655901588 /loss/FBt1/detection_loss‚2.9309210046974723 .loss/FBt3/detection_loss‚3.765505507514805 _step‚55 0loss/FRt1/assignment_loss‚7.3923837911515005 /loss/FRt3/assignment_loss‚7.221761712488139 2loss/SRqqt1/assignment_loss‚3.5917076693640815 1loss/SRqqt4/detection_loss‚2.0590757547932514 epoch‚0 trainer/global_step‚699 0loss/FBt3/assignment_loss‚0.4305479713964949 .loss/FRt2/detection_loss‚3.538581973976559 0loss/SRqqt2/detection_loss‚4.272438008534281 %loss/total_loss‚87.45297973265264 0loss/FBt1/assignment_loss‚0.2726037961671109 /loss/FRt1/detection_loss‚3.7523721080450785 % _timestamp‚1.7884454105427327e+09 _runtime‚ 526.014234356 1loss/SRqqt3/assignment_loss‚2.707485795021057 1loss/SRqqt4/assignment_loss‚2.760960892719381 .loss/FBt4/detection_loss‚8.644195234775543 /loss/FRt4/assignment_loss‚7.3548538313971637­Xò:“ž‚(:ˆ ð…æÔ©—êý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.297' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.6594 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0´§ˆó:Ÿ‚(:¯ ð…æÔÆåŸÿ disk.sda.in‚0 cpu‚25.108692686857637# disk.nvme0n1.out‚1674.607421875 proc.cpu.threads‚17* disk./.usagePercent‚19.903410712600454 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.out‚ 2.83984375 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.drbd2266.out‚0( proc.memory.availableMB‚ 1964.7578125 network.sent‚858618 disk.drbd1716.in‚0% disk.nvme0c0n1.out‚1674.607421875 disk.dm-0.out‚1674.607421875 disk.sda.out‚20.21875% disk./.usageGB‚296.57520294189453 disk.dm-1.in‚0 disk.dm-1.out‚20.21875 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125# proc.memory.rssMB‚ 2550.94921875 disk.rbd1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0* proc.memory.percent‚15.569758415222168 disk.sda3.out‚20.21875 disk.rbd0.in‚0$ memory_percent‚88.00806999206543 network.recv‚270974 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0+üùJ“ ‚(:ˆ ÿ…æÔ¨’ý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚190.329' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.0658 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚03.êF¡‚(:» ÿ…æÔ®¨™ÿ& disk.nvme0c0n1.out‚1732.5458984375 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 20.58984375 disk.drbd2080.out‚0) proc.memory.availableMB‚ 1964.74609375 cpu‚25.107190542514594 disk.rbd1.out‚ 2.9970703125 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 20.58984375! disk.dm-0.out‚1732.5458984375$ memory_percent‚88.00814151763916) disk./.usagePercent‚19.90410602906345 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.rbd0.in‚0 proc.cpu.threads‚17 network.sent‚907792 network.recv‚275893 disk.sda1.in‚0 disk.drbd2265.in‚0 disk.drbd2266.out‚0! proc.memory.rssMB‚ 2550.953125* proc.memory.percent‚15.569782257080078% disk./.usageGB‚296.58556365966797$ disk.nvme0n1.out‚1732.5458984375 disk.sda3.in‚0 disk.sda3.out‚ 20.58984375 disk.nvme0c0n1.in‚0.078125qÅǪ0¢+ _wandbruntime‚562  _runtime‚562}ÞdfÄ£¾ _runtime‚ 562.378363275 _step‚56 & lr-AdamW/pg1‚7.848435390698755e-05 & lr-AdamW/pg2‚7.848435390698755e-05 trainer/global_step‚749 % _timestamp‚1.7884454469067996e+096ˆ2>Ȥ _step‚56 & lr-AdamW/pg1‚7.848435390698755e-05 & lr-AdamW/pg2‚7.848435390698755e-05 trainer/global_step‚749 % _timestamp‚1.7884454469067996e+09 _runtime‚ 562.3783632758 óÙo0¥+ _wandbruntime‚562  _runtime‚562Z½Üo¸'¦²O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3462812 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10519596 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚468 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚41193798347776 _step‚57 _runtime‚ 562.379433646 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚42896883763712 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚42896210165248 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚42281691621888 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24424064 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13885926 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6131702 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9594514 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6131700 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3462346 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10519596 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 614536546304 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚42281673618944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 612278002370 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 156002304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 615192141824 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚46905115699200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9594046 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179942400 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚41806730780354 trainer/global_step‚749 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 615192141824 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944IæÈT짶O TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10519598 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13904466 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚46258684619264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚466 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 646587082240 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 615192141824 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚46905115699200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9594046 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179942400 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚41806730780354 trainer/global_step‚749 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 615192141824 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3462812 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10519596 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_startç¤?iÐ /allocation.small_pool.peak‚22630 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚468 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚41193798347776 _step‚57 _runtime‚ 562.379433646 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚42896883763712 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚42896210165248 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚42281691621888 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚24424064 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13885926 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6131702 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9594514 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6131700 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3462346 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10519596 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 614536546304 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚42281673618944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 612278002370 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 156002304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13904380 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚46320335356928 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚41861779876310 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6139870 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚46967815030784 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚58 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10533612 trainer/global_step‚749 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13904380 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10533614 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13922946 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚46320359297024 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10533612 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24437992 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 647455733760 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42338044660736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚24437992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚ 563.083334274 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 616011582976 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3467004 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 160183296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚41248702478336 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚24456560 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚42338044660736 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13922946 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚478 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9606872 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚46967630907392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚41248685438976 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 613094437334 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚42954056243712 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 647295550464 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚42954056243712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚42338026657792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚42338026657792 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6139868 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚476 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 615355974144 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚42953382631936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 615355974144 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24456560 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚41862451346086 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 % _timestamp‚1.7884454476111522e+09 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9607350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 184123392 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903þ÷ Æ&ªÀM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10533612 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24437992 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 647455733760 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42338044660736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚24437992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚ 563.083334274 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 616011582976 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3467004 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 160183296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚41248702478336 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚24456560 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚42338044660736 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13922946 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚478 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9606872 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚46967630907392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚41248685438976 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 613094437334 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚42954056243712 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 647295550464 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚42954056243712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚42338026657792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚42338026657792 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6139868 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚476 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 615355974144 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚42953382631936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 615355974144 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24456560 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚41862451346086 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 % _timestamp‚1.7884454476111522e+09 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9607350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 184123392 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3467480 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 616011582976 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 613748867750 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13904380 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚46320335356928 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚41861779876310 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6139870 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚46967815030784 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚58 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10533612 trainer/global_step‚749 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚13904380 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10533614 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚13922946 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚46320359297024:p)qž0«+ _wandbruntime‚563  _runtime‚563¦·Ø+M¬Ç epoch‚0 0loss/SRqqt2/detection_loss‚3.572980785234408 /loss/FBt1/detection_loss‚2.3311381039313144 .loss/FBt3/detection_loss‚4.860843187857654 %loss/total_loss‚86.64808161042207 0loss/FBt1/assignment_loss‚0.2415746953379615 .loss/FRt3/detection_loss‚3.013406555669812 1loss/SRqqt4/detection_loss‚2.1756144065049385 0loss/FBt3/assignment_loss‚0.5598060219349934 0loss/FBt4/assignment_loss‚0.5962901219564418 0loss/SRqqt1/detection_loss‚2.207583687746078 1loss/SRqqt3/detection_loss‚1.4106989462356865 .loss/FBt2/detection_loss‚7.376315154135227 trainer/global_step‚749 1loss/SRqqt2/assignment_loss‚2.783805772662163 2loss/SRqqt4/assignment_loss‚2.8397476327034736 -loss/FRt2/detection_loss‚3.94368863850832 % _timestamp‚1.7884454476129324e+09 _runtime‚ 563.084431616 /loss/FRt2/assignment_loss‚7.272105783224106 /loss/FRt3/assignment_loss‚7.170319465490488 2loss/SRqqt3/assignment_loss‚2.6785530469503747 _step‚59 .loss/FRt1/assignment_loss‚7.37464501063029 /loss/FRt4/assignment_loss‚7.375170461062727 2loss/SRqqt1/assignment_loss‚3.4041295732770647 0loss/FBt2/assignment_loss‚0.2915454172041791 .loss/FRt1/detection_loss‚3.421219665805499 /loss/FRt4/detection_loss‚2.8288590540659837 .loss/FBt4/detection_loss‚6.918040422292886#a1Q­Ë 0loss/FBt3/assignment_loss‚0.5598060219349934 0loss/FBt4/assignment_loss‚0.5962901219564418 0loss/SRqqt1/detection_loss‚2.207583687746078 1loss/SRqqt3/detection_loss‚1.4106989462356865 .loss/FBt2/detection_loss‚7.376315154135227 trainer/global_step‚749 1loss/SRqqt2/assignment_loss‚2.783805772662163 2loss/SRqqt4/assignment_loss‚2.8397476327034736 -loss/FRt2/detection_loss‚3.94368863850832 % _timestamp‚1.7884454476129324e+09 _runtime‚ 563.084431616 /loss/FRt2/assignment_loss‚7.272105783224106 /loss/FRt3/assignment_loss‚7.170319465490488 2loss/SRqqt3/assignment_loss‚2.6785530469503747 _step‚59 .loss/FRt1/assignment_loss‚7.37464501063029 /loss/FRt4/assignment_loss‚7.375170461062727 2loss/SRqqt1/assignment_loss‚3.4041295732770647 0loss/FBt2/assignment_loss‚0.2915454172041791 .loss/FRt1/detection_loss‚3.421219665805499 /loss/FRt4/detection_loss‚2.8288590540659837 .loss/FBt4/detection_loss‚6.918040422292886 epoch‚0 0loss/SRqqt2/detection_loss‚3.572980785234408 /loss/FBt1/detection_loss‚2.3311381039313144 .loss/FBt3/detection_loss‚4.860843187857654 %loss/total_loss‚86.64808161042207 0loss/FBt1/assignment_loss‚0.2415746953379615 .loss/FRt3/detection_loss‚3.013406555669812 1loss/SRqqt4/detection_loss‚2.1756144065049385;?b?|ž®‚(:“ Ž†æÔü¯‘ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚126.319' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.263800000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0qƒ˜½C¯‚(:¸ Ž†æÔ»‹ÿ disk.sda3.in‚0! disk.dm-0.out‚1767.2841796875 disk.sda.in‚0 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125* proc.memory.percent‚15.569806098937988 network.sent‚914871 disk.drbd2080.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 2.9970703125$ disk.nvme0n1.out‚1767.2841796875 disk.dm-1.out‚ 21.1640625 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 21.1640625 disk.sda1.out‚0) proc.memory.availableMB‚ 1964.30078125# proc.memory.rssMB‚ 2550.95703125 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0$ memory_percent‚88.01085948944092 cpu‚25.10535087221759 proc.cpu.threads‚17& disk.nvme0c0n1.out‚1767.2841796875 disk.sda3.out‚ 21.1640625 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.dm-1.in‚0 disk.drbd1716.in‚0 network.recv‚281567 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0) disk./.usagePercent‚19.90369308891072 disk.drbd2080.out‚0$ disk./.usageGB‚296.5794105529785ó¤Õž°‚(:“ †æÔÖúþ gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚175.938' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.187599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÒÀA>±‚(:³ †æÔÚ²£€ disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚1804.20166015625 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 3.0595703125 disk.drbd1716.out‚0# memory_percent‚88.0108118057251# proc.memory.rssMB‚ 2550.95703125 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.dm-1.in‚0 cpu‚25.104305588676656 disk.nvme0n1.in‚0.078125* disk./.usagePercent‚19.904251953357967 disk.rbd0.out‚0 disk.sda2.in‚0 disk.drbd2265.out‚0 proc.cpu.threads‚17 network.sent‚964740% disk.nvme0n1.out‚1804.20166015625$ disk./.usageGB‚296.5877380371094 disk.drbd2266.in‚0 disk.sda.in‚0 disk.rbd1.in‚0 disk.dm-1.out‚21.8125 disk.drbd2265.in‚0) proc.memory.availableMB‚ 1964.30859375* proc.memory.percent‚15.569806098937988 network.recv‚286860 disk.sda3.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda3.out‚21.8125" disk.dm-0.out‚1804.20166015625 disk.sda.out‚21.8125 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd1716.in‚0.BÀM0²+ _wandbruntime‚599  _runtime‚599 MåÁij¾ _runtime‚ 599.359437515 _step‚60 & lr-AdamW/pg1‚8.372362986873573e-05 & lr-AdamW/pg2‚8.372362986873573e-05 trainer/global_step‚799 % _timestamp‚1.7884454838878813e+09.OÊ*È´Â & lr-AdamW/pg1‚8.372362986873573e-05 & lr-AdamW/pg2‚8.372362986873573e-05 trainer/global_step‚799 % _timestamp‚1.7884454838878813e+09 _runtime‚ 599.359437515 _step‚60DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 158099456 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚43939004875776 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚26050916 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚50030793241088 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6540102 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14830518 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚45099343564288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11220396 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 656177947648 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 QIDeviceStatsMoµùã6)nitor.on_train_batch_start/inactive_split.small_pool.current‚466 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 653113341778 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7884454838885894e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11220398 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚45099325561344 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6540100 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3694379 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 689571738624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 655522352128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚45755521511936 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14811978 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚799 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚49341245442560 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚45754847913472 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚44592101178194 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26032374 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 655522352128 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 689729838080 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚45099325561344 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 656177947648 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚50030975280640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 182039552 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10234947 _runtime‚ 599.360424828 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚45754847913472 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3694845 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚44592772647762 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 653767771986 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _step‚61 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11220398 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14830518 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚43938987836416Ã’DL¾'·¸O =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚45754847913472 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚44592101178194 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚26032374 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 655522352128 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 689729838080 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚45099325561344 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 656177947648 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚50030975280640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 182039552 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10234947 _runtime‚ 599.360424828 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚45754847913472 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3694845 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚44592772647762 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 653767771986 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _step‚61 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11220398 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14830518 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚43938987836416 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚45755521511936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚45099343564288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚468 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 158099456 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚43939004875776 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚26050916 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚50030793241088 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6540102 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14830518 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚45099343564288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11220396 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 656177947648 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚466 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 653113341778 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7884454838885894e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11220398 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚45099325561344 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6540100 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3694379 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 689571738624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 655522352128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚45755521511936 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14811978 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚799 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚49341245442560 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11234414 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10247250 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14848998 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14830432 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚463 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚49402872240128 _step‚62 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚44648493213494 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚45812020380160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10247713 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 654584207158 trainer/global_step‚799 _runtime‚ 600.054053037 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 653929776742 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 656341779968 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26083412 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚49402896180224 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26064844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 690447447552 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚45155696603136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 656997388800 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 656997388800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚26064844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6548270 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26083412 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚45155678600192 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14848998 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3698982 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚50093499616768 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14830432 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚44647821743718 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 656341779968 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11234412 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155988992 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚43993909006336 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚45155696603136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11234414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3699443 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179929088 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11234412 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11234414 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10247250 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14848998 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚14830432 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚463 ^KDeviceSÊš’‡¬tatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚49402872240128 _step‚62 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚44648493213494 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚45812020380160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10247713 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 654584207158 trainer/global_step‚799 _runtime‚ 600.054053037 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 653929776742 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 656341779968 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26083412 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚49402896180224 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26064844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 690447447552 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚45155696603136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 656997388800 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 656997388800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚26064844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6548270 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26083412 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚45155678600192 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚14848998 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3698982 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚50093499616768 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14830432 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0>€lM‚0½+ _wandbruntime‚600  _runtime‚6000#çÑP¾Ê 1loss/SRqqt2/detection_loss‚3.8425893262028694 /loss/FBt3/detection_loss‚3.6679889175580955 2loss/SRqqt1/assignment_loss‚2.8944127474512373 .loss/FRt2/detection_loss‚4.431206980347634 epoch‚0 % _timestamp‚1.7884454845836668e+09 .loss/FBt4/detection_loss‚5.092563688755035 /loss/FRt3/assignment_loss‚7.217712163925171 1loss/SRqqt3/assignment_loss‚2.660623073577881 1loss/FBt2/assignment_loss‚0.44008060432159773 /loss/FRt4/detection_loss‚2.9168534681752876 0loss/SRqqt3/detection_loss‚2.029892246662068 1loss/SRqqt4/detection_loss‚1.7950851522489555 _runtime‚ 600.055185013 1loss/FBt1/assignment_loss‚0.24904736447228507 0loss/FRt1/assignment_loss‚7.3526952120722555 -loss/FRt2/assignment_loss‚7.3702627658844 1loss/SRqqt2/assignment_loss‚2.837576824426651 0loss/FBt3/assignment_loss‚0.7029938593808766 /loss/SRqqt1/detection_loss‚2.11841129899594 $loss/total_loss‚81.4592142664616 trainer/global_step‚799 1loss/FBt4/assignment_loss‚0.39520859718322754 _step‚63 0loss/FRt4/assignment_loss‚7.3852487228534835 2loss/SRqqt4/assignment_loss‚2.7624730658200054 .loss/FRt1/detection_loss‚3.158837779444091 .loss/FRt3/detection_loss‚3.042222676643481 /loss/FBt1/detection_loss‚1.9276787643963456 .loss/FBt2/detection_loss‚5.167548965662718ÃF¸T¿Î /loss/FBt3/detection_loss‚3.6679889175580955 2loss/SRqqt1/assignment_loss‚2.8944127474512373 .loss/FRt2/detection_loss‚4.431206980347634 epoch‚0 % _timestamp‚1.7884454845836668e+09 .loss/FBt4/detection_loss‚5.092563688755035 /loss/FRt3/assignment_loss‚7.217712163925171 1loss/SRqqt3/assignment_loss‚2.660623073577881 1loss/FBt2/assignment_loss‚0.44008060432159773 /loss/FRt4/detection_loss‚2.9168534681752876 0loss/SRqqt3/detection_loss‚2.029892246662068 1loss/SRqqt4/detection_loss‚1.7950851522489555 _runtime‚ 600.055185013 1loss/FBt1/assignment_loss‚0.24904736447228507 0loss/FRt1/assignment_loss‚7.3526952120722555 -loss/FRt2/assignment_loss‚7.3702627658844 1loss/SRqqt2/assignment_loss‚2.837576824426651 0loss/FBt3/assignment_loss‚0.7029938593808766 /loss/SRqqt1/detection_loss‚2.11841129899594 $loss/total_loss‚81.4592142664616 trainer/global_step‚799 1loss/FBt4/assignment_loss‚0.39520859718322754 _step‚63 0loss/FRt4/assignment_loss‚7.3852487228534835 2loss/SRqqt4/assignment_loss‚2.7624730658200054 .loss/FRt1/detection_loss‚3.158837779444091 .loss/FRt3/detection_loss‚3.042222676643481 /loss/FBt1/detection_loss‚1.9276787643963456 .loss/FBt2/detection_loss‚5.167548965662718 1loss/SRqqt2/detection_loss‚3.8425893262028694?ŠqŸ“À‚(:ˆ »†æÔδŽý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚131.521' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.3042 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 TäòIÁ‚(:¾ »†æÔË¿˜ÿ disk.sda.out‚ 22.8203125 disk.rbd1.in‚0$ disk.nvme0n1.out‚1877.6337890625$ memory_percent‚88.01147937774658) disk./.usagePercent‚19.90594390715176 disk.drbd1716.in‚0 disk.drbd1716.out‚0! disk.dm-0.out‚1877.6337890625 disk.sda3.in‚0 disk.sda1.in‚0& disk.nvme0c0n1.out‚1877.6337890625 network.sent‚ 1.009111e+06$ disk./.usageGB‚296.6129493713379 disk.drbd2080.out‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.dm-1.out‚ 22.8203125 disk.rbd0.out‚0 disk.sda2.in‚0 disk.sda3.out‚ 22.8203125 disk.drbd2265.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0) proc.memory.availableMB‚ 1964.19921875* proc.memory.percent‚15.569829940795898 cpu‚25.101864183020865 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.dm-1.in‚0" proc.memory.rssMB‚ 2550.9609375 disk.drbd2080.in‚0 disk.sda.in‚0 disk.rbd1.out‚ 3.5048828125 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17 network.recv‚296831 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125R©„“‚(:ˆ ʆæÔ¹´÷ü gpu.0.gpu‚41.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.693' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.1386 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0| &2GÂ(:¼ ʆæÔÿ„âþ network.sent‚ 1.048887e+06 disk.sda.in‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0$ disk./.usageGB‚296.6203727722168! disk.dm-0.out‚1911.3642578125 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.dm-0.in‚0.078125 network.recv‚302674 disk.rbd0.out‚0 disk.sda1.in‚0$ disk.nvme0n1.out‚1911.3642578125 disk.dm-1.in‚0$ memory_percent‚88.01157474517822) proc.memory.availableMB‚ 1964.18359375 disk.drbd2266.in‚0 disk.rbd1.out‚ 3.5048828125 disk.dm-1.out‚ 23.671875* proc.memory.percent‚15.569829940795898 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda.out‚ 23.671875 disk.sda2.in‚0" proc.memory.rssMB‚ 2550.9609375 cpu‚25.100338689772165 disk.sda3.out‚ 23.671875 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 proc.cpu.threads‚17 disk.drbd1716.out‚0* disk./.usagePercent‚19.906442097813393& disk.nvme0c0n1.out‚1911.3642578125 ßÝ0Ä+ _wandbruntime‚636  _runtime‚636b˜´ÃŽ & lr-AdamW/pg1‚8.896290583048388e-05 & lr-AdamW/pg2‚8.896290583048388e-05 trainer/global_step‚849 % _timestamp‚1.7884455211055334e+09 _runtime‚ 636.57714861 _step‚64›ªûõÇÆÁ & lr-AdamW/pg2‚8.896290583048388e-05 trainer/global_step‚849 % _timestamp‚1.7884455211055334e+09 _runtime‚ 636.57714861 _step‚64 & lr-AdamW/pg1‚8.896290583048388e-05@¤ÂVº0Ç+ _wandbruntime‚636  _runtime‚636.1/¹'ȳO _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚47916995506688 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11921196 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3927446 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15756570 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _step‚65 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚ 636.578193863 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 696508157952 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15756570 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚48613485661696 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 697163753472 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 145516544 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚48613485661696 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚27677768 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚432 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 732867627008 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚47378814515170 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚46684194364416 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚47916977503744 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚48614159260160 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 694603111394 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10876378 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10875946 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15738030 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚47916977503744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11921198 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11921196 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15738030 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚47378143045602 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚52423758385664 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11921198 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚52423782325760 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 693948681186 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3927876 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚53156626012672 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 trainer/global_step‚849 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 697163753472 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚47916995506688Ò?)Z½'É·O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11921198 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚52423782325760 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 693948681186 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3927876 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚53156626012672 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 trainer/global_step‚849 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 697163753472 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚47916995506688 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚47916995506688 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11921196 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3927446 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15756570 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _step‚65 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚ 636.578193863 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 696508157952 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15756570 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚48613485661696 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 697163753472 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 145516544 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚48613485661696 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚27677768 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚432 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 732867627008 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚47378814515170 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚46684194364416 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚47916977503744 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚48614159260160 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 694603111394 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10876378 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10875946 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15738030 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚47916977503744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11921198 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11921196 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15738030 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚47378143045602 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚52423758385664Apuð0Ê+ _wandbruntime‚637  _runtime‚637ô¡ÄÊ˾M I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚47973330542592 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15775050 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 162280448 % _timestamp‚1.7884455217928364e+09 Þ_ìú$J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11935212 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 _step‚66 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27691696 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 trainer/global_step‚849 _runtime‚ 637.264855386 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27710264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15775050 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 186220544 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚47434535080902 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 694765116150 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11935214 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚48671331740160 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 695419546566 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11935214 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6956668 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15756484 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10888735 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15756484 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27710264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚47433863611126 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚52485433063424 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430õ<[ È&ÌÂM NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚15756484 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10888735 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15756484 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27710264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚47433863611126 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚52485433063424 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚47973330542592 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15775050 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 162280448 % _timestamp‚1.7884455217928364e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11935212 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 _step‚66 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27691696 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 trainer/global_step‚849 _runtime‚ 637.264855386 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27710264 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15775050 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 186220544 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚47434535080902 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 694765116150 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11935214 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚48671331740160 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 695419546566 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11935214 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6956668B 3k50Í+ _wandbruntime‚637  _runtime‚637u _OÎÉ /loss/FRt1/assignment_loss‚7.224656679413536 /loss/FRt4/assignment_loss‚7.312301023801168 2loss/SRqqt3/assignment_loss‚2.7610395463073956 epoch‚0 /loss/FRt3/assignment_loss‚7.224450224921817 1loss/FBt2/assignment_loss‚0.46946105916293546 1loss/FBt3/assignment_loss‚0.49009222960915266 /loss/FRt1/detection_loss‚3.5938155011019925 /loss/SRqqt3/detection_loss‚1.54740388850674 trainer/global_step‚849 /loss/FRt2/assignment_loss‚7.182031345367432 /loss/FRt4/detection_loss‚2.4793398504145445 _step‚67 0loss/FBt1/assignment_loss‚0.2382629869892948 1loss/SRqqt2/assignment_loss‚2.629821774634448 0loss/SRqqt4/assignment_loss‚2.39462248980999 0loss/FBt4/assignment_loss‚0.5111410294014674 1loss/SRqqt1/detection_loss‚3.7409958536536467 0loss/SRqqt2/detection_loss‚3.455792386423458 _runtime‚ 637.266088514 1loss/SRqqt1/assignment_loss‚3.472994800271659 .loss/FBt1/detection_loss‚3.524494968775048 % _timestamp‚1.7884455217945411e+09 .loss/FRt3/detection_loss‚2.277259810672452 1loss/SRqqt4/detection_loss‚2.3929145147100956 %loss/total_loss‚81.88619874576631 /loss/FRt2/detection_loss‚3.2307440042495728 .loss/FBt2/detection_loss‚7.574789146059437 /loss/FBt3/detection_loss‚2.1291147646267103 .loss/FBt4/detection_loss‚4.028658866882324?üË·SÏÍ 1loss/SRqqt2/assignment_loss‚2.629821774634448 0loss/SRqqt4/assignment_loss‚2.39462248980999 0loss/FBt4/assignment_loss‚0.5111410294014674 1loss/SRqqt1/detection_loss‚3.7409958536536467 0loss/SRqqt2/detection_loss‚3.455792386423458 _runtime‚ 637.266088514 1loss/SRqqt1/assignment_loss‚3.472994800271659 .loss/FBt1/detection_loss‚3.524494968775048 % _timestamp‚1.7884455217945411e+09 .loss/FRt3/detection_loss‚2.277259810672452 1loss/SRqqt4/detection_loss‚2.3929145147100956 %loss/total_loss‚81.88619874576631 /loss/FRt2/detection_loss‚3.2307440042495728 .loss/FBt2/detection_loss‚7.574789146059437 /loss/FBt3/detection_loss‚2.1291147646267103 .loss/FBt4/detection_loss‚4.028658866882324 /loss/FRt1/assignment_loss‚7.224656679413536 /loss/FRt4/assignment_loss‚7.312301023801168 2loss/SRqqt3/assignment_loss‚2.7610395463073956 epoch‚0 /loss/FRt3/assignment_loss‚7.224450224921817 1loss/FBt2/assignment_loss‚0.46946105916293546 1loss/FBt3/assignment_loss‚0.49009222960915266 /loss/FRt1/detection_loss‚3.5938155011019925 /loss/SRqqt3/detection_loss‚1.54740388850674 trainer/global_step‚849 /loss/FRt2/assignment_loss‚7.182031345367432 /loss/FRt4/detection_loss‚2.4793398504145445 _step‚67 0loss/FBt1/assignment_loss‚0.2382629869892948CµÝELžÐ‚(:“ Ù†æÔª·‘ý gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚140.334' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.066800000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0.òIÑ‚(:¾ Ù†æԹɴÿ network.recv‚306393% disk.nvme0n1.out‚1953.48876953125 disk.drbd1716.in‚0 disk.sda3.out‚ 24.19140625 disk.dm-0.in‚0.078125$ memory_percent‚88.01193237304688$ proc.memory.availableMB‚1964.125" proc.memory.rssMB‚ 2550.9609375 disk.rbd1.in‚0 disk.sda1.in‚0 cpu‚25.09953960413588 proc.cpu.threads‚17$ disk./.usageGB‚296.6388053894043 disk.dm-1.out‚ 24.19140625 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0) disk./.usagePercent‚19.90767912621884 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚1953.48876953125" disk.dm-0.out‚1953.48876953125 disk.sda1.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda.in‚0* proc.memory.percent‚15.569829940795898 network.sent‚ 1.054317e+06 disk.rbd1.out‚ 3.5673828125 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.rbd0.in‚0 disk.sda.out‚ 24.19140625íì.ó“Ò‚(:ˆ è†æÔ­ê¢ý gpu.0.gpu‚45.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚39.0 gpu.0.powerWatts‚186.523' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.3046 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0µ<4öQÓ‚(:Æ è†æÔ‚ΰÿ% disk.nvme0n1.out‚1989.61767578125 disk.drbd1716.out‚0' disk.nvme0c0n1.out‚1989.61767578125 disk.sda3.in‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.sda3.out‚ 24.87890625 cpu‚25.098826079310182 disk.rbd0.in‚0 disk.sda.out‚ 24.87890625 network.recv‚312945* disk./.usagePercent‚19.908192421324994 disk.drbd2266.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd2080.out‚0$ memory_percent‚88.01195621490479 proc.cpu.threads‚17 disk.sda1.out‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 24.87890625 disk.drbd2265.out‚0 disk.sda.in‚0$ disk./.usageGB‚296.6464538574219" disk.dm-0.out‚1989.61767578125 disk.sda2.out‚0# proc.memory.rssMB‚ 2550.96484375* proc.memory.percent‚15.569853782653809 disk.rbd1.out‚ 3.6611328125) proc.memory.availableMB‚ 1964.12109375 network.sent‚ 1.104742e+06v!™D0Ô+ _wandbruntime‚673  _runtime‚673çV$¶ÃÕ½ & lr-AdamW/pg1‚9.420218179223206e-05 & lr-AdamW/pg2‚9.420218179223206e-05 trainer/global_step‚899 $ _timestamp‚1.788445558095858e+09 _runtime‚ 673.567444105 _step‚68q\ˆúÇÖÁ _step‚68 & lr-AdamW/pg1‚9.420218179223206e-05 & lr-AdamW/pg2‚9.420218179223206e-05 trainer/global_step‚899 $ _timestamp‚1.788445558095858e+09 _runtime‚ 673.567444105DÙ<ÿ0×+ _wandbruntime‚673  _runtime‚673ÓžÎkUسO TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16664082 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚51472123409920 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚56282386057728 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚69 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4160277 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 186233856 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12621998 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11517179 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚51472797008384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚49429400892416 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 738149559296 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚899 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 737493963776 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚50734647449088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚29304620 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16664082 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚51472797008384 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚51472123409920 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚478 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚50164856382578 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12621996 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 737493963776 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 162293760 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7356902 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7356900 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 735438450802 $ _timestamp‚1.788445558096745e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚50734629446144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚55506319208960 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚50164184913010 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16682622 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚50734647449088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414œú̽'Ù·O Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚51472123409920 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚478 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚50164856382578 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12621996 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 737493963776 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 162293760 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7356902 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7356900 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 735438450802 $ _timestamp‚1.788445558096745e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚50734629446144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚55506319208960 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚50164184913010 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16682622 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚50734647449088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16664082 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚51472123409920 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚56282386057728 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚69 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4160277 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 186233856 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12621998 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11517179 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚51472797008384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚49429400892416 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 738149559296 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚899 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 737493963776 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚50734647449088 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚29304620 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16664082 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚51472797008384 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0E+UŽÁ0Ú+ _wandbruntime‚674  _runtime‚674ØGÜÄ&Û¾M G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29318548 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11529994 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚51529295876608 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12636014 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 738969000448 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚49484322062336 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚49484305022976 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚899 _step‚70 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884455587871203e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55567946006528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12636014 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚51529969488384 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚50790982484992 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 738969000448 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4164924 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚50220576948310 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚51529969488384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 738313391616 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11529525 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 777122755072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚29337116 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12636012 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 735600455558 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16682536 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16682536 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55567969946624 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16701102 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 175734784 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚467 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 151794688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12636012 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚56344916966912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29337116 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416Ôv–¹È$ÜÂM S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚51529969488384 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚50790982484992 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 738969000448 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4164924 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚50220576948310 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚51529969488384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 738313391616 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11529525 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 777122755072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚29337116 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12636012 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 735600455558 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16682536 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16682536 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55567969946624 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16701102 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 175734784 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚467 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 151794688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12636012 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚56344916966912 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29337116 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29318548 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11529994 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚51529295876608 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12636014 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 738969000448 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚49484322062336 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚49484305022976 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚899 _step‚70 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884455587871203e+09 E=DeviceStatsMonitor.on_trainʹzÉ_batch_end/inactive_split.all.peak‚904 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55567946006528 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12636014 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844FQc0Ý+ _wandbruntime‚674  _runtime‚674<â 5MÞÇ /loss/FRt1/assignment_loss‚6.178493291681463 /loss/FRt4/assignment_loss‚7.423929877397491 0loss/SRqqt1/detection_loss‚2.509823889753963 0loss/SRqqt3/detection_loss‚1.404167669215801 .loss/FBt4/detection_loss‚8.344670989240209 epoch‚0 2loss/SRqqt1/assignment_loss‚2.6857157454771152 0loss/FBt3/assignment_loss‚0.3500139599803106 1loss/SRqqt4/detection_loss‚2.2855320131262906 .loss/FBt2/detection_loss‚4.624166865544073 _step‚71 /loss/FRt2/assignment_loss‚7.071750618162609 1loss/SRqqt2/assignment_loss‚2.633587463482006 2loss/SRqqt3/assignment_loss‚2.8312041586951207 2loss/SRqqt4/assignment_loss‚2.7095852136611938 1loss/FBt1/assignment_loss‚0.32908636839701183 trainer/global_step‚899 0loss/FBt2/assignment_loss‚0.4072207815448125 0loss/FBt4/assignment_loss‚1.0158415585756302 %loss/total_loss‚83.47427069276608 % _timestamp‚1.7884455587886164e+09 _runtime‚ 674.260127528 .loss/FRt1/detection_loss‚2.663261885263703 0loss/SRqqt2/detection_loss‚4.036861265027845 /loss/FRt3/assignment_loss‚7.130368482871134 .loss/FBt3/detection_loss‚6.416466899310637 -loss/FRt2/detection_loss‚2.68634942032042 .loss/FRt3/detection_loss‚2.128137872294813 .loss/FRt4/detection_loss‚3.698107682350205 /loss/FBt1/detection_loss‚1.9099267213922353€í­QßË .loss/FRt1/detection_loss‚2.663261885263703 0loss/SRqqt2/detection_loss‚4.036861265027845 /loss/FRt3/assignment_loss‚7.130368482871134 .loss/FBt3/detection_loss‚6.416466899310637 -loss/FRt2/detection_loss‚2.68634942032042 .loss/FRt3/detection_loss‚2.128137872294813 .loss/FRt4/detection_loss‚3.698107682350205 /loss/FBt1/detection_loss‚1.9099267213922353 /loss/FRt1/assignment_loss‚6.178493291681463 /loss/FRt4/assignment_loss‚7.423929877397491 0loss/SRqqt1/detection_loss‚2.509823889753963 0loss/SRqqt3/detection_loss‚1.404167669215801 .loss/FBt4/detection_loss‚8.344670989240209 epoch‚0 2loss/SRqqt1/assignment_loss‚2.6857157454771152 0loss/FBt3/assignment_loss‚0.3500139599803106 1loss/SRqqt4/detection_loss‚2.2855320131262906 .loss/FBt2/detection_loss‚4.624166865544073 _step‚71 /loss/FRt2/assignment_loss‚7.071750618162609 1loss/SRqqt2/assignment_loss‚2.633587463482006 2loss/SRqqt3/assignment_loss‚2.8312041586951207 2loss/SRqqt4/assignment_loss‚2.7095852136611938 1loss/FBt1/assignment_loss‚0.32908636839701183 trainer/global_step‚899 0loss/FBt2/assignment_loss‚0.4072207815448125 0loss/FBt4/assignment_loss‚1.0158415585756302 %loss/total_loss‚83.47427069276608 % _timestamp‚1.7884455587886164e+09 _runtime‚ 674.260127528GUÑÌožà‚(:“ ÷†æÔ‰–ý gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.662' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.132400000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 öMá‚(: ÷†æÔà¸îÿ disk.drbd2080.out‚0 disk.rbd1.in‚0 disk.sda3.out‚ 25.17578125 disk.dm-0.in‚0.078125$ memory_percent‚88.01372051239014 disk.rbd1.out‚ 3.8486328125 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125# proc.memory.rssMB‚ 2550.96484375& disk.nvme0c0n1.out‚2035.8974609375 network.sent‚ 1.10995e+06 network.recv‚316370 disk.drbd2265.in‚0 disk.sda.out‚ 25.17578125$ disk.nvme0n1.out‚2035.8974609375 disk.sda1.out‚0 disk.sda3.in‚0* disk./.usagePercent‚19.908488622042107 disk.drbd2265.out‚0 disk.drbd2266.out‚0! disk.dm-0.out‚2035.8974609375 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 25.17578125$ disk./.usageGB‚296.6508674621582 disk.drbd2080.in‚0 disk.drbd2266.in‚0) proc.memory.availableMB‚ 1963.83203125* proc.memory.percent‚15.569853782653809 cpu‚25.097396008878004 proc.cpu.threads‚17 disk.drbd1716.in‚0kbkQ“â‚(:ˆ †‡æÔ…¸’ý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.413' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6826 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0k€¦nGã‚(:¼ †‡æÔÉ´ÿ disk.rbd1.out‚ 3.8486328125 disk.sda3.out‚ 25.76171875 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.dm-0.out‚ 2073.85546875 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0# proc.memory.rssMB‚ 2550.96484375 cpu‚25.0964400772064 disk.sda2.out‚0" disk.nvme0n1.out‚ 2073.85546875 disk.rbd0.out‚0 disk.sda.out‚ 25.76171875 disk.sda1.in‚0 network.recv‚322602 disk.dm-1.out‚ 25.76171875$ disk.nvme0c0n1.out‚ 2073.85546875 disk.sda3.in‚0 disk.drbd2266.in‚0* proc.memory.percent‚15.569853782653809% disk./.usageGB‚296.65974044799805 disk.drbd1716.out‚0 disk.sda.in‚0$ memory_percent‚88.01562786102295) proc.memory.availableMB‚ 1963.51953125 proc.cpu.threads‚17 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 network.sent‚ 1.160262e+06* disk./.usagePercent‚19.909084095566755’…›žä‚(:“ •‡æÔŸ¡Œý gpu.0.gpu‚32.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.837' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.367399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Õ¦÷"Oå‚(:Ä •‡æÔí ÿ disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.sda.out‚ 26.18359375 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚2116.13916015625 disk.dm-1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 26.18359375 proc.cpu.threads‚17 disk.drbd2265.out‚0 disk.rbd1.out‚ 3.9111328125' proc.memory.availableMB‚ 1964.015625 cpu‚25.09550193063543 network.recv‚325562 disk.drbd2080.in‚0 disk.sda3.out‚ 26.18359375 network.sent‚ 1.165377e+06% disk./.usageGB‚296.67199325561523# proc.memory.rssMB‚ 2550.96484375* proc.memory.percent‚15.569853782653809' disk.nvme0c0n1.out‚2116.13916015625 disk.sda3.in‚0 disk.sda.in‚0$ memory_percent‚88.01259994506836 disk.rbd1.in‚0% disk.nvme0n1.out‚2116.13916015625 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0* disk./.usagePercent‚19.909906391766732 disk.drbd2265.in‚0jÈ;ƒ0æ+ _wandbruntime‚710  _runtime‚710ß@]Äç¾ & lr-AdamW/pg1‚9.944145775398023e-05 & lr-AdamW/pg2‚9.944145775398023e-05 trainer/global_step‚949 % _timestamp‚1.7884455952682655e+09 _runtime‚ 710.739859368 _step‚729ד]Èè & lr-AdamW/pg1‚9.944145775398023e-05 & lr-AdamW/pg2‚9.944145775398023e-05 trainer/global_step‚949 % _timestamp‚1.7884455952682655e+09 _runtime‚ 710.739859368 _step‚72Hº©»0é+ _wandbruntime‚710  _runtime‚710`¨ƒº'ê´O UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚52950898249986 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 776273790210 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚54330761158144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚497 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 778479769600 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17590134 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚54331434756608 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7765300 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚17608674 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13322796 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 196719616 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 819221993984 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12156984 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13322798 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884455952690113e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17608674 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4391682 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 819394773504 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚52174624459776 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 775619360002 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7765302 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚53552281388544 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 779135365120 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚59408054146048 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚58588832152064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13322796 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚54330761158144 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13322796 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚54330761158144 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚52950898249986 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 776273790210 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚54330761158144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚904 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚497 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 778479769600 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚17590134 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚54331434756608 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7765300 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚17608674 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13322796 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 196719616 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 819221993984 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12156984 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414IÄ9 0ì+ _wandbruntime‚711  _runtime‚711þb=Jní¾M UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7773470 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 779954806272 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚54388607236608 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 779299197440 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚52229511550976 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚54388607236608 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚479 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 777090225382 _step‚74 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚54387933624832 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚52229528590336 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚53006618815718 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚53005947345942 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚30945400 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 779954806272 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30963968 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17627154 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚53608634427392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚53608634427392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13336812 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚59470574795776 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17608588 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 158086144 _runtime‚ 711.420347839 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 779299197440 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚59470756822016 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17608588 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12169198 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4395730 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 820091906048 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30963968 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13336814 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13336812 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17627154 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚53608652430336 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0¯îrÈ&îÂM J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚54388607236608 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚479 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 777090225382 _step‚74 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚54387933624832 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚52229528590336 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚53006618815718 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚53005947345942 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚30945400 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 779954806272 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30963968 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17627154 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚53608634427392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚53608634427392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13336812 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚59470574795776 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17608588 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 158086144 _runtime‚ 711.420347839 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 779299197440 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚59470756822016 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17608588 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12169198 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4395730 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚885 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 820091906048 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30963968 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13336814 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13336812 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17627154 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚53608652430336 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7773470 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 779954806272 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚54388607236608 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 779299197440 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚52229511550976 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚57189398906368 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚56369951333888 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14023596 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚470 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8173700 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14023598 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 817109129618 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚56369933330944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18534726 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚62533961392640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚62533766770176 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚61671369035264 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚56369933330944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884456322235286e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542Þ5û¸O TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18534726 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚62533961392640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allo¼m[‰&cated_bytes.all.peak‚ 9954819072 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚62533766770176 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚61671369035264 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚56369933330944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884456322235286e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚55736940117394 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚32558324 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 862568417280 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚18534726 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 819465575424 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚54919830987776 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8173702 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚885 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚57190072504832 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚18516186 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚57189398906368 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚56369951333888 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14023596 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚470 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8173700 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14023598 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 817109129618 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚56369933330944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2Mò‘AÜ0ü+ _wandbruntime‚748  _runtime‚748Ms'>Ä&ý¾M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚61733043713024 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚54974718078976 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚57247244984832 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14037612 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚999 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12810417 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 817925564790 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8181868 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 817271134374 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚56426304372736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚ 748.393679307 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32572252 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚57247244984832 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚56426286369792 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚55791989213350 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 820940612096 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32590820 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 820285003264 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18534640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚475 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚62596459435520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198803456 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚61733019772928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4628549 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 863415722496 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚57246571373056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18553206 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12810892 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18534640 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚54974718078976 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚57247244984832 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14037612 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚999 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12810417 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 817925564790 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8181868 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 817271134374 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚56426304372736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚ 748.393679307 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32572252 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚57247244984832 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚56426286369792 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚55791989213350 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 820940612096 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32590820 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 820285003264 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18534640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚475 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚904 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚62596459435520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198803456 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚61733019772928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4628549 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 863415722496 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚57246571373056 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18553206 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12810892 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18534640 ODeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚60048036654592 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8582100 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19442238 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19460778 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19460778 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚59187603276288 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚60048710253056 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 905792368128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13438317 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8582102 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14724396 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14724396 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 860451381248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13438317 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8582102 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14724396 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14724396 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 860451381248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚60048036654592 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8582100 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19442238 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19460778 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19460778 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚59187603276288 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚60048710253056 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 905792368128Q¶ Äš0Ž+ _wandbruntime‚786  _runtime‚786æ^¡¿M K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34217672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚65722052076544 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13450734 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884456707873387e+09 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 861926417920 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8590270 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4860918 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13451188 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚34199104 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚59243956315136 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 861270809088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8590268 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _step‚82 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _runtime‚ 786.259398627 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34199104 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚58578702550534 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚65722238297088 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚64815556656128 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 162280448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19460692 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14738412 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14738412 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚1049 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19479258 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚64815580596224 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚58578031080758 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19479258 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚60105882733056 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4860466 ;4DeviceStatsMonitor.on_traiP¥%6® n_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 861270809088 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14738414 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 861926417920 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚60105882733056 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚59243938312192 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 906495420416 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 186220544 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚57719924606976 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚59243956315136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 858106473782 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34217672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚59243938312192 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚60105209121280 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952‰íý¶É&ÃM OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13451188 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚34199104 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚59243956315136 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 861270809088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8590268 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _step‚82 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _runtime‚ 786.259398627 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34199104 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚58578702550534 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚65722238297088 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚64815556656128 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 162280448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19460692 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14738412 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14738412 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚1049 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19479258 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚64815580596224 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚58578031080758 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19479258 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚60105882733056 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4860466 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 861270809088 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14738414 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 861926417920 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚60105882733056 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚59243938312192 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 906495420416 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 186220544 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚57719924606976 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚59243956315136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 858106473782 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34217672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚59243938312192 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚60105209121280 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34217672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚65722052076544 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13450734 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884456707873387e+09 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 861926417920 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8590270 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4860918RsÂÕƒ0‘+ _wandbruntime‚786  _runtime‚786ô½‘N’È epoch‚0 trainer/global_step‚1049 /loss/FRt1/assignment_loss‚5.859626104434331 /loss/FRt1/detection_loss‚3.0447497653464475 /loss/FRt2/detection_loss‚3.4237350085202385 0loss/SRqqt2/detection_loss‚3.561876798803742 1loss/SRqqt3/detection_loss‚1.8534940507666786 1loss/SRqqt3/assignment_loss‚2.793443098435035 0loss/FBt2/assignment_loss‚0.5545509739000114 /loss/FBt4/assignment_loss‚0.536646559833186 1loss/SRqqt4/detection_loss‚1.6538815030562026 /loss/FRt2/assignment_loss‚6.502247501822079 1loss/SRqqt1/assignment_loss‚2.492246424450594 /loss/FBt3/assignment_loss‚0.432435044675367 /loss/FRt3/detection_loss‚3.6996634297072886 %loss/total_loss‚78.21964858007323 % _timestamp‚1.7884456707890818e+09 _runtime‚ 786.260575689 _step‚83 0loss/FBt1/assignment_loss‚0.4632544812022843 /loss/FRt3/assignment_loss‚5.537717413902283 1loss/SRqqt2/assignment_loss‚2.857923108178216 1loss/SRqqt4/assignment_loss‚2.740991140263421 /loss/FBt2/detection_loss‚3.5200513049557403 /loss/FRt4/assignment_loss‚6.034067595446551 1loss/SRqqt1/detection_loss‚3.3981105140684282 .loss/FBt1/detection_loss‚2.484302154635162 /loss/FRt4/detection_loss‚2.7712873689554356 .loss/FBt3/detection_loss‚3.603263472765684 -loss/FBt4/detection_loss‚8.40008376194882ÈæÍSR“Ì _runtime‚ 786.260575689 _step‚83 0loss/FBt1/assignment_loss‚0.4632544812022843 /loss/FRt3/assignment_loss‚5.537717413902283 1loss/SRqqt2/assignment_loss‚2.857923108178216 1loss/SRqqt4/assignment_loss‚2.740991140263421 /loss/FBt2/detection_loss‚3.5200513049557403 /loss/FRt4/assignment_loss‚6.034067595446551 1loss/SRqqt1/detection_loss‚3.3981105140684282 .loss/FBt1/detection_loss‚2.484302154635162 /loss/FRt4/detection_loss‚2.7712873689554356 .loss/FBt3/detection_loss‚3.603263472765684 -loss/FBt4/detection_loss‚8.40008376194882 epoch‚0 trainer/global_step‚1049 /loss/FRt1/assignment_loss‚5.859626104434331 /loss/FRt1/detection_loss‚3.0447497653464475 /loss/FRt2/detection_loss‚3.4237350085202385 0loss/SRqqt2/detection_loss‚3.561876798803742 1loss/SRqqt3/detection_loss‚1.8534940507666786 1loss/SRqqt3/assignment_loss‚2.793443098435035 0loss/FBt2/assignment_loss‚0.5545509739000114 /loss/FBt4/assignment_loss‚0.536646559833186 1loss/SRqqt4/detection_loss‚1.6538815030562026 /loss/FRt2/assignment_loss‚6.502247501822079 1loss/SRqqt1/assignment_loss‚2.492246424450594 /loss/FBt3/assignment_loss‚0.432435044675367 /loss/FRt3/detection_loss‚3.6996634297072886 %loss/total_loss‚78.21964858007323 % _timestamp‚1.7884456707890818e+09SÔ…yâ“”‚(:ˆ ï‡æÔèó¨ý gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.441' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4882 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚02F»ëQ•‚(:Æ ï‡æÔÿµ¨ÿ network.recv‚351465 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.sda1.out‚0) proc.memory.availableMB‚ 1963.78515625* proc.memory.percent‚15.569853782653809 network.sent‚ 1.286795e+06 disk.sda2.in‚0$ memory_percent‚88.01400661468506 cpu‚25.090832912159794 proc.cpu.threads‚17% disk./.usageGB‚296.69126892089844 disk.drbd2080.in‚0 disk.drbd2265.out‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚2348.44970703125 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.dm-0.in‚0.078125* disk./.usagePercent‚19.911199997837247 disk.dm-1.out‚ 29.17578125 disk.sda3.out‚ 29.17578125 disk.rbd0.in‚0 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚2348.44970703125 disk.sda3.in‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda.out‚ 29.17578125 disk.drbd2266.out‚0" disk.dm-0.out‚2348.44970703125# proc.memory.rssMB‚ 2550.96484375 disk.rbd1.out‚ 4.404296875 disk.dm-1.in‚0 disk.sda2.out‚0Íì7ž–‚(:“ þ‡æÔíƒý gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚146.471' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.294199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Y,_O—‚(:Ä þ‡æÔ¯¬•ÿ* disk./.usagePercent‚19.911480582094775% disk./.usageGB‚296.69544982910156 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚2386.89111328125 disk.rbd1.out‚ 4.404296875 disk.sda3.in‚0 disk.rbd0.in‚0" disk.dm-0.out‚2386.89111328125 disk.rbd1.in‚0 disk.drbd2265.in‚0) proc.memory.availableMB‚ 1963.76953125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0# proc.memory.rssMB‚ 2550.96484375 network.sent‚ 1.33706e+06 disk.drbd2080.in‚0 disk.sda3.out‚ 29.81640625 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.sda2.in‚0* proc.memory.percent‚15.569853782653809 cpu‚25.090112470389933 disk.dm-1.in‚0 disk.drbd2080.out‚0 disk.sda.in‚0 disk.sda1.in‚0% disk.nvme0n1.out‚2386.89111328125 disk.drbd1716.out‚0 disk.drbd2265.out‚0 proc.cpu.threads‚17 network.recv‚357452 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.dm-1.out‚ 29.81640625 disk.sda.out‚ 29.81640625# memory_percent‚88.0141019821167à×00˜+ _wandbruntime‚822  _runtime‚822ÈÇ™Á ' lr-AdamW/pg1‚0.00011515928563922473 ' lr-AdamW/pg2‚0.00011515928563922473 trainer/global_step‚1099 % _timestamp‚1.7884457073952155e+09 _runtime‚ 822.866796304 _step‚84œ”ªËšÅ _runtime‚ 822.866796304 _step‚84 ' lr-AdamW/pg1‚0.00011515928563922473 ' lr-AdamW/pg2‚0.00011515928563922473 trainer/global_step‚1099 % _timestamp‚1.7884457073952155e+09TOÊ»¨0›+ _wandbruntime‚822  _runtime‚822$~§E»'œµO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 948868385280 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20386830 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚477 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15425198 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8990500 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚67836466741760 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5087676 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚62907348001280 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚62906674402816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15425198 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚60410244043776 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚62005237215744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 898779808434 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚60410227004416 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚35812028 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15425196 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14078655 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20368290 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 164390912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 898125378226 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20386830 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚85 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚61308352382642 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 trainer/global_step‚1099 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15425196 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚62005237215744 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚67836442801664 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20368290 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 902092782592 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚479 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 949032776192 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 901437187072 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14078176 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚62005255218688 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8990502 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚62906674402816 iMÆÒ¹O C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚62906674402816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 948868385280 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20386830 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚477 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚15425198 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8990500 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚67836466741760 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5087676 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚62907348001280 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚62906674402816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15425198 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 _LDeviceStatsMonitor.on_oK®ítrain_batch_start/requested_bytes.large_pool.allocated‚60410244043776 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 185637376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚62005237215744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863048704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 898779808434 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚60410227004416 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚35812028 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15425196 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14078655 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20368290 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 164390912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 898125378226 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚22903 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚20386830 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚85 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚61308352382642 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 trainer/global_step‚1099 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15425196 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚62005237215744 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚67836442801664 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20368290 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 902092782592 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚479 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 949032776192 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 901437187072 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14078176 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚62005255218688 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8990502 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889U1Z²0ž+ _wandbruntime‚823  _runtime‚823þí·ÿÅ&Ÿ¿M ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚61364744417942 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 949737844736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35844524 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚62061608257536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚62061590254592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14091479 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚67898117479424 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35825956 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5092809 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 158086144 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 823.615873167 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20405310 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14091021 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 182026240 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 899596243606 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚62061608257536 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20405310 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8998670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 _step‚86 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚62964520481280 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚62963846869504 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8998668 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚60465148174336 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944X+ÅÉ& ÃM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚35844524 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 trainer/global_step‚1099 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14091479 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚67898117479424 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35825956 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5092809 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 158086144 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 823.615873167 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20405310 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14091021 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 182026240 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 899596243606 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚62061608257536 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20405310 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8998670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 _step‚86 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚62964520481280 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚62963846869504 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8998668 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚60465148174336 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚61364744417942 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 949737844736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35844524 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚62061608257536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚62061590254592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚65765312151040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21294342 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 943078588416 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16125996 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 992217750016 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5320423 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 175748096 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 _step‚89 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚71911045626880 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚70919003624960 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚63155433532416 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚64822907161088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 992065942016 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21312882 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9398900 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚65765985749504 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21312882 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 _runtime‚ 860.170366445 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚16125998 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21294342 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚64095065719618 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚65765985749504 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14719787 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚63155450571776 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 938960717634 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 trainer/global_step‚1149 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚37420338 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16125998 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚71911221374976>¿§q¾'¯¸O \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚64822907161088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 992065942016 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21312882 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9398900 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚65765985749504 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21312882 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 _runtime‚ 860.170366445 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚16125998 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21294342 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚64095065719618 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚65765985749504 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14719787 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚63155450571776 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 938960717634 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 trainer/global_step‚1149 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚37420338 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16125998 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚71911221374976 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚70918979684864 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚65765312151040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚21294342 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 943078588416 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16125996 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 992217750016 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643691477 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5320423 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 175748096 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 _step‚89 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚71911045626880 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚70919003624960 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚63155433532416 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0Yà›j0°+ _wandbruntime‚860  _runtime‚860[ff*Å&±¿M R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚64150114815574 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚71973740571648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚90 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚64879260199936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9407070 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚65823158229504 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5325019 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 943898029568 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37471376 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14732087 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 993086209024 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚70980654362624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚64879260199936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚70980630422528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 860.876780946 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚71973550156800 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚37452808 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21331362 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚469 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 943898029568 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 943242420736 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚64150786285350 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚65823158229504 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21312796 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚64879242196992 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚471 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 166474752 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0f¸XRë²ÃM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37471376 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚37452808 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚64150114815574 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚71973740571648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_seg§ÅoýÞ"ments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚90 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚64879260199936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9407070 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚65823158229504 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5325019 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 943898029568 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37471376 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14732087 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 993086209024 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚70980654362624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚64879260199936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚70980630422528 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 860.876780946 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚71973550156800 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚37452808 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21331362 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚469 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 943898029568 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 943242420736 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚64150786285350 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚65823158229504 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚21312796 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚64879242196992 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚471 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 166474752 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _runtime‚ 898.142240276 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚68624623497728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16826796 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚1199 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚67640559103488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚67640541100544 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5552019 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚66881107587026 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚66880436117458 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884457826703825e+09 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 984064394240 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚67640559103488 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚68623949899264 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 983408798720 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1035145711616 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚65900657099776 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22220394 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39065732 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚478 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚65900640060416 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 980450487250 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 164390912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39047190 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16826796 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22220394 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 979796057042 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 984064394240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 983408798720 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚68623949899264 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚75036850610688 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9807300\{\l¿»O 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚18542 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 188331008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚18540 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39065732 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚478 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚65900640060416 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 980450487250 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654430208 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 164390912 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39047190 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚16826796 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚18540 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22220394 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 979796057042 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 984064394240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9954819072 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863048704 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9954819072 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 983408798720 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚68623949899264 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚22630 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚75036850610688 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 673598464 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9807300 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚74001540508160 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚74001516568064 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚480 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671469568 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _runtime‚ 898.142240276 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚68624623497728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16826796 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚1199 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861845973 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚67640559103488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 93þºê»U 26034944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚67640541100544 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5552019 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 673598464 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 655595520 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚66881107587026 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚66880436117458 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884457826703825e+09 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 984064394240 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚67640559103488 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚22630 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚68623949899264 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 655595520 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 983408798720 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚22903 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1035145711616 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚65900657099776 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚18542 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22220394 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚65955544190976 trainer/global_step‚1199 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15372096 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚66936156682982 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚65955561230336 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 984228226560 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15372574 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚68681795977728 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16840812 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚478 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 981266922422 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22257414 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚67696894139392 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚75099364015616 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚74063191245824 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22238848 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39098228 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 186220544 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚67696894139392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22238848 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16840814 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚476 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 162280448 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 655608832 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚74063167305728 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 984883835392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884457833933535e+09 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16840812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 984228226560 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚67696912142336 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22257414 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1036010489344 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39098228 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9815468 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚68681122365952 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚39079660 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚66936828152758 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 980612492006 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚74063167305728 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 984883835392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863048704 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884457833933535e+09 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861845973 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16840812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 673611776 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 655608832 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 984228226560 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚67696912142336 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 673611776 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22257414 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1036010489344 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚39098228 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9815468 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚68681122365952 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9954819072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚18566 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚22630 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚39079660 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚66936828152758 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654430416 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 980612492006 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚18568 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9954819072 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671469776 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚65955544190976 trainer/global_step‚1199 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15372096 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚18566 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 185637376 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚66936156682982 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚65955561230336 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 984228226560 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚22903 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15372574 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚22903 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚68681795977728 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16840812 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚478 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 981266922422 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22257414 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚67696894139392 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚75099364015616 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚74063191245824 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22238848 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39098228 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 186220544 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643691477 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚67696894139392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863048704 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22238848 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16840814 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚476 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 162280448^} d¼0Ã+ _wandbruntime‚898  _runtime‚898¹ˆ³ãNÄÈ 1loss/SRqqt4/assignment_loss‚2.322136587518103 /loss/FRt2/assignment_loss‚6.117281653664329 1loss/FBt3/assignment_loss‚0.20623686007869432 .loss/FRt3/detection_loss‚2.481644876978614 1loss/SRqqt3/detection_loss‚0.9174971800742102 %loss/total_loss‚71.55209390633782 % _timestamp‚1.7884457833950558e+09 2loss/SRqqt2/assignment_loss‚2.5409724160476967 0loss/FBt4/assignment_loss‚0.6594853890793664 .loss/FBt4/detection_loss‚4.487549236844566 trainer/global_step‚1199 1loss/SRqqt1/assignment_loss‚2.769141709363019 /loss/FBt2/assignment_loss‚0.389735938719988 .loss/FRt4/detection_loss‚2.241768291637079 1loss/SRqqt2/detection_loss‚3.8925104637940726 _runtime‚ 898.866573548 /loss/FRt1/assignment_loss‚6.045927386130056 0loss/SRqqt1/detection_loss‚3.976856436994341 0loss/SRqqt4/detection_loss‚1.813356232966351 0loss/FRt3/assignment_loss‚3.3694389083168725 1loss/SRqqt3/assignment_loss‚2.593317906061808 -loss/FRt2/detection_loss‚3.00190194086595 /loss/FBt1/detection_loss‚5.3441367607727575 .loss/FBt3/detection_loss‚2.541367347330548 epoch‚0 /loss/FRt4/assignment_loss‚4.987419710230472 1loss/FBt1/assignment_loss‚0.38839026050515746 .loss/FBt2/detection_loss‚4.421923752341952 _step‚95 .loss/FRt1/detection_loss‚4.042096660021813!¿îRÅÌ .loss/FRt1/detection_loss‚4.042096660021813 1loss/SRqqt4/assignment_loss‚2.322136587518103 /loss/FRt2/assignment_loss‚6.117281653664329 1loss/FBt3/assignment_loss‚0.20623686007869432 .loss/FRt3/detection_loss‚2.481644876978614 1loss/SRqqt3/detection_loss‚0.9174971800742102 %loss/total_loss‚71.55209390633782 % _timestamp‚1.7884457833950558e+09 2loss/SRqqt2/assignment_loss‚2.5409724160476967 0loss/FBt4/assignment_loss‚0.6594853890793664 .loss/FBt4/detection_loss‚4.487549236844566 trainer/global_step‚1199 1loss/SRqqt1/assignment_loss‚2.769141709363019 /loss/FBt2/assignment_loss‚0.389735938719988 .loss/FRt4/detection_loss‚2.241768291637079 1loss/SRqqt2/detection_loss‚3.8925104637940726 _runtime‚ 898.866573548 /loss/FRt1/assignment_loss‚6.045927386130056 0loss/SRqqt1/detection_loss‚3.976856436994341 0loss/SRqqt4/detection_loss‚1.813356232966351 0loss/FRt3/assignment_loss‚3.3694389083168725 1loss/SRqqt3/assignment_loss‚2.593317906061808 -loss/FRt2/detection_loss‚3.00190194086595 /loss/FBt1/detection_loss‚5.3441367607727575 .loss/FBt3/detection_loss‚2.541367347330548 epoch‚0 /loss/FRt4/assignment_loss‚4.987419710230472 1loss/FBt1/assignment_loss‚0.38839026050515746 .loss/FBt2/detection_loss‚4.421923752341952 _step‚95_8$ÛgÆ‚(:’ ؈æÔÀ‡Âý gpu.0.gpu‚42.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚172.531' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚34.50620000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¯ÇÊVDÇ‚(:¹ ؈æÔðÐÊ€ disk.drbd2266.out‚0 disk.sda2.in‚0 disk.dm-1.out‚33.03125 disk.drbd2265.out‚0 proc.cpu.threads‚17* disk./.usagePercent‚19.915465851380297 disk.sda3.in‚0 disk.sda.out‚33.03125 disk.rbd1.in‚0* proc.memory.percent‚15.569877624511719 network.sent‚ 1.459828e+06 network.recv‚385530 disk.drbd1716.in‚0 disk.drbd1716.out‚0' disk.nvme0c0n1.out‚2608.79931640625 disk.sda3.out‚33.03125 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.rbd1.out‚ 4.7724609375 cpu‚25.085609142915533" disk.dm-0.out‚2608.79931640625 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚2608.79931640625 disk.drbd2265.in‚0% disk./.usageGB‚296.75483322143555 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda2.out‚0 disk.drbd2080.in‚0 proc.memory.rssMB‚ 2550.96875 disk.sda1.in‚0 disk.dm-1.in‚0$ memory_percent‚88.01698684692383' proc.memory.availableMB‚ 1963.296875Œ¿>È‚(:’ çˆæÔžèýü gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚123.409' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.681800000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0jb NÉ‚(:à çˆæÔàƒ¡ÿ disk.sda3.in‚0" disk.dm-0.out‚2682.44873046875 disk.sda3.out‚ 33.48828125 disk.dm-0.in‚0.078125 disk.sda.out‚ 33.48828125 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 cpu‚25.134364281222307 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 network.recv‚389672$ disk./.usageGB‚296.7685432434082 disk.rbd1.in‚0) proc.memory.percent‚16.13600254058838 disk.rbd1.out‚ 5.0458984375 disk.dm-1.in‚0' disk.nvme0c0n1.out‚2682.44873046875' proc.memory.availableMB‚ 1870.390625 proc.cpu.threads‚17% disk.nvme0n1.out‚2682.44873046875 disk.dm-1.out‚ 33.48828125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0# proc.memory.rssMB‚ 2643.72265625 network.sent‚ 1.508678e+06* disk./.usagePercent‚19.916385942458355 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0$ memory_percent‚88.58404159545898 disk.drbd1716.in‚0ÈUœÊ‚(:’ öˆæÔ×Æ£ý gpu.0.gpu‚9.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚123.409' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.681800000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0æotEË‚(:º öˆæÔíº‡ÿ disk.sda.out‚ 34.140625 disk.sda3.out‚ 34.140625 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚2720.03076171875 disk.drbd1716.in‚0 cpu‚25.21098443199887' disk.nvme0c0n1.out‚2720.03076171875$ disk./.usageGB‚296.7763214111328 disk.sda.in‚0 disk.sda2.out‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0" disk.dm-0.out‚2720.03076171875 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 5.0458984375 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0$ memory_percent‚88.48438262939453* proc.memory.percent‚16.036701202392578 network.recv‚394779 disk.drbd2080.in‚0! proc.memory.rssMB‚ 2627.453125 disk.drbd2265.in‚0 network.sent‚ 1.515335e+06 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.drbd2266.out‚0* disk./.usagePercent‚19.916907941820668 disk.dm-1.out‚ 34.140625 disk.drbd2265.out‚0& proc.memory.availableMB‚ 1886.71875 proc.cpu.threads‚17 disk.dm-0.in‚0.078125¹ˆv“Ì‚(:ˆ …‰æÔïùúü gpu.0.gpu‚23.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚188.661' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.7322 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0^¬&MÍ‚(: …‰æÔ¶žŽ€ disk.drbd2080.out‚0 disk.sda3.out‚ 34.53515625 disk.sda.out‚ 34.53515625 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.sda.in‚0 disk.dm-1.out‚ 34.53515625 disk.drbd2265.out‚0 network.recv‚398130 disk.sda1.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 cpu‚25.280558736431086 disk.dm-1.in‚0 disk.sda3.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 proc.cpu.threads‚17 network.sent‚ 1.520494e+06$ disk./.usageGB‚296.7874412536621 disk.drbd2080.in‚0" disk.dm-0.out‚2757.46240234375 disk.drbd1716.out‚0( proc.memory.availableMB‚ 1885.4765625) disk./.usagePercent‚19.91765420378298 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 5.1083984375% disk.nvme0n1.out‚2757.46240234375$ memory_percent‚88.49196434020996* proc.memory.percent‚16.037273406982422' disk.nvme0c0n1.out‚2757.46240234375 disk.drbd2266.out‚0! proc.memory.rssMB‚ 2627.546875m‹×BxÎbs PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated"trainer/global_step2:JPšÑP;rÏbm JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed"trainer/global_step2:JPO@qªhÐbc @DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak"trainer/global_step2:JP&A†}yÑbt QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak"trainer/global_step2:JP«ªngÒbb ?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections"trainer/global_step2:JPq㌈lÓbg DDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed"trainer/global_step2:JP¨öEoÔbj GDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak"trainer/global_step2:JP2ÄwÕbr ODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated"trainer/global_step2:JP%(Jý_Öba >DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries"trainer/global_step2:ã„¡<JPúí“t×bo LDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current"trainer/global_step2:JP¾è¶ zØbu RDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed"trainer/global_step2:JP,uN eÙb` =DeviceStatsMonitor.on_validation_batch_start/num_device_alloc"trainer/global_step2:JP°ú9‹tÚbo LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak"trainer/global_step2:JPºøé÷mÛbh EDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current"trainer/global_step2:JP†òµ‚mÜbh EDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak"trainer/global_step2:JPÀ§¥tÝbo LDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed"trainer/global_step2:JP†úð·wÞbr ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current"trainer/global_step2:JPЃÖOkßbf CDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak"trainer/global_step2:JP“u8wàbr ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current"trainer/global_step2:JP-èm)sábn KDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak"trainer/global_step2:JP4âx+nâbi FDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current"trainer/global_step2:JP 1rpãbk HDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current"trainer/global_step2:JP})cºsäbn KDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak"trainer/global_step2:JPÝáànåbi FDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed"trainer/global_step2:JP‹§xõqæbl IDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated"trainer/global_step2:JP–&¬bmçbh EDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed"trainer/global_step2:JP¶«/xèbs PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated"trainer/global_step2:JP£2Emébh EDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated"trainer/global_step2:JP-,šCmêbh EDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.peak"trainer/global_step2:JPæ oëbj GDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed"trainer/global_step2:JP# ¿mìbh EDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed"trainer/global_step2:JP9 ZÊpíbk HDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed"trainer/global_step2:JP¾º»…wîbr ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current"trainer/global_step2:JPús Önïbi FDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current"trainer/global_step2:JPüö1$jðbe BDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak"trainer/global_step2:JPÝÐwÎtñbo LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak"trainer/global_step2:JP?*ßròbm JDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current"trainer/global_step2:JPcê,,uóbp MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current"trainer/global_step2:JPÿ¢zôbu RDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed"trainer/global_step2:JP[ âkõbf CDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak"trainer/global_step2:JP!UA1söbn KDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak"trainer/global_step2:JPšŽ žo÷bj GDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current"trainer/global_step2:JPl—®Êsøbn KDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated"trainer/global_step2:JPh#™eùb` =DeviceStatsMonitor.on_validation_batch_start/segment.all.peak"trainer/global_step2:JP:V„|gúbb ?DeviceStatsMonitor.on_validation_batch_start/active.all.current"trainer/global_step2:JPˆlâtûbo LDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current"trainer/global_step2:JP¤Ó1÷uübp MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed"trainer/global_step2:JPX+¦Utýbo LDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed"trainer/global_step2:JP–Ibyþbt QDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated"trainer/global_step2:JP&hôuÿbp MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed"trainer/global_step2:JP™þ0øv€bq NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated"trainer/global_step2:JPRˆoñnbi FDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed"trainer/global_step2:JPgm1y‚bt QDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated"trainer/global_step2:JPùnÚmƒbh EDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed"trainer/global_step2:JP113x„bs PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated"trainer/global_step2:JPîHn…bi FDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak"trainer/global_step2:JP7IH»p†bk HDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated"trainer/global_step2:JP“Êקt‡bo LDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated"trainer/global_step2:JPúu;Óvˆbq NDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current"trainer/global_step2:JP¾†‚d‰b_ ~w«br ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current"trainer/global_step2:JPMð×ço¬bj GDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current"trainer/global_step2:JPÃIŽ‹o­bj GDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated"trainer/global_step2:JP¡làr®bm JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed"trainer/global_step2:JP>Qèðp¯bk HDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed"trainer/global_step2:JPL$†zs°bn KDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak"trainer/global_step2:JPÙög|±bw TDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current"trainer/global_step2:JP²Ðl‘q²bl IDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated"trainer/global_step2:JP74·5l³bg DDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed"trainer/global_step2:JP*n}Wu´bp MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed"trainer/global_step2:JP®DeviceStatsMonitor.on_validation_batch_start/segment.all.freed"trainer/global_step2:JP·ôt»bo LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak"trainer/global_step2:JPB#Ïo¼bj GDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak"trainer/global_step2:JP»lÑOx½bs PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated"trainer/global_step2:JP(]p¾bk HDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated"trainer/global_step2:JP ÿÎNc¿b^ ;DeviceStatsMonitor.on_validation_batch_start/max_split_size"trainer/global_step2:JPðìšyÀbt QDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated"trainer/global_step2:JP ú<lÁbg DDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak"trainer/global_step2:JPÊ01kÂbf CDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak"trainer/global_step2:JP˜OyélÃbg DDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.peak"trainer/global_step2:JPB^ÆoÄbj GDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current"trainer/global_step2:JP—¬kqÅbl IDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak"trainer/global_step2:JPa½Œd|Æbw TDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current"trainer/global_step2:JP­BýŠ]ÇbX 5DeviceStatsMonitor.on_validation_batch_start/num_ooms"trainer/global_step2:JPOAЪmÈbh EDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed"trainer/global_step2:JPdºç»0É+ _wandbruntime‚955  _runtime‚955TÊA[,*ʦT GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510430720 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚10207676 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 aNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚70008760903168 NADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚40262510 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚69185540332806 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 ZGDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated‚71018925754368 \JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚ 1009672423424 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚17496450 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 VIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚16039405 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 `MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚68179075580928 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 NADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚40277703 dQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚70008760903168 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863048704 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 \PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚5831727 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 1006447712518 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚22630 LDDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.peak‚908 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863048704 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 111552000 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ]JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚70008742900224 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚22903 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 QDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚17496450 QGDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak‚22630 bODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚77535980766208 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492427776 eRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚76473934886400 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510430720 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9954819072 XLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚5831573 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490911116 LCDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak‚2605 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492427776 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507950476 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 YFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚69185032382330 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚ 955.761991201 REDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚40277703 TEDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.peak‚ 9643691477 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚22766060 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9954819072 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861845973 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 135492096 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 1061910387712 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚156 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15193 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 1009672423424 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 dQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚68179092620288 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 1005956801402 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 1010164851200 YFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚71018415323648 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 1010164851200 REDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚16039249 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚10207678 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚22781251 YLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚17496452 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚154 trainer/global_step‚1208 _step‚96 FDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863048704 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 \PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚5831727 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 1006447712518 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚22630 LDDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.peak‚908 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863048704 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 111552000 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ]JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚70008742900224 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚22903 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 QDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚17496450 QGDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak‚22630 bODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚77535980766208 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492427776 eRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚76473934886400 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510430720 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9954819072 XLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚5831573 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490911116 LCDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak‚2605 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492427776 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507950476 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 YFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚69185032382330 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚ 955.761991201 REDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚40277703 TEDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.peak‚ 9643691477 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚22766060 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9954819072 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861845973 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 135492096 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 1061910387712 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚156 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_validation_batch_st˜OÅÖ÷ art/segment.all.peak‚844 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15193 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 1009672423424 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 dQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚68179092620288 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 1005956801402 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 1010164851200 YFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚71018415323648 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 1010164851200 REDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚16039249 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚10207678 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚22781251 YLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚17496452 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚154 trainer/global_step‚1208 _step‚96 FDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak"trainer/global_step2:JP‡»é~wÜbr ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak"trainer/global_step2:JPa‘NÚjÝbe BDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak"trainer/global_step2:JP8cårÞbm JDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak"trainer/global_step2:JPòÎØsßbn KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed"trainer/global_step2:JPkÔžõpàbk HDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current"trainer/global_step2:JPÓC;pábk HDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current"trainer/global_step2:JPôÊlnâbi FDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed"trainer/global_step2:JPbB¹šqãbl IDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.allocated"trainer/global_step2:JPÿA9Psäbn KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed"trainer/global_step2:JPBÌÂoåbj GDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak"trainer/global_step2:JPŸ1Õª|æbw TDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated"trainer/global_step2:JP|•È’uçbp MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current"trainer/global_step2:JPKòø÷eèb` =DeviceStatsMonitor.on_validation_batch_end/active.all.current"trainer/global_step2:JPÔî·hébc @DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak"trainer/global_step2:JP‹YX©qêbl IDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak"trainer/global_step2:JP‡cÖÛfëba >DeviceStatsMonitor.on_validation_batch_end/segment.all.current"trainer/global_step2:JP,9lìbg DDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current"trainer/global_step2:JPžëÐCgíbb ?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams"trainer/global_step2:JPƒ®Çuîbp MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.current"trainer/global_step2:JP\«:xmïbh EDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current"trainer/global_step2:JP@Vp=pðbk HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated"trainer/global_step2:JP¹{ë%pñbk HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed"trainer/global_step2:JP’BÆzròbm JDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed"trainer/global_step2:JP&+Þjóbe BDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak"trainer/global_step2:JPbähôbc @DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated"trainer/global_step2:JP%lHoõbj GDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak"trainer/global_step2:JPÍÊK‰uöbp MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current"trainer/global_step2:JPlOþžn÷bi FDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed"trainer/global_step2:JPJ|øbw TDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated"trainer/global_step2:JPΕ±aùb\ 9DeviceStatsMonitor.on_validation_batch_end/max_split_size"trainer/global_step2:JPÈÏtúbo LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated"trainer/global_step2:JPå^³pûbk HDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed"trainer/global_step2:JPUõ­rübm JDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak"trainer/global_step2:JPjRqPjýbe BDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak"trainer/global_step2:JPƒm­Õkþbf CDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed"trainer/global_step2:JP#÷[Áoÿbj GDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated"trainer/global_step2:JP>ÀwPs€bn KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.freed"trainer/global_step2:JPqBŠ7obj GDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated"trainer/global_step2:JP6ê¸r‚bm JDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated"trainer/global_step2:JPÈîÅ|rƒbm JDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.freed"trainer/global_step2:JP^éÎc„b^ ;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak"trainer/global_step2:JPPçöqj…be BDeviceStatsMonitor.on_validation_batch_end/active.large_pool.freed"trainer/global_step2:JP‘aøSx†bs PDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.freed"trainer/global_step2:JP ôr‡bm JDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak"trainer/global_step2:JPŒiVGmˆbh EDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current"trainer/global_step2:JPWÊVÑz‰bu RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current"trainer/global_step2:JPb0‰õlŠbg DDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed"trainer/global_step2:JPþú*•s‹bn KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed"trainer/global_step2:JP*ºc¤vŒbq NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated"trainer/global_step2:JP«è¢wbr ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak"trainer/global_step2:JP özlŽbg DDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak"trainer/global_step2:JPWötbo LDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current"trainer/global_step2:JP[yÆcb^ ;DeviceStatsMonitor.on_validation_batch_end/active.all.freed"trainer/global_step2:JPÔ¶«Öj‘be BDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak"trainer/global_step2:JPøû]0n’bi FDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated"trainer/global_step2:JPD·’­t“bo LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated"trainer/global_step2:JPß t”bo LDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current"trainer/global_step2:JP¥½Míl•bg DDeviceStatsMonitor.on_validation_batch_end/oversize_segments.current"trainer/global_step2:JPùh±g–bb ?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated"trainer/global_step2:JP²® ûm—bh EDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.freed"trainer/global_step2:JP•ÁÙ£j˜be BDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed"trainer/global_step2:JP¤=û•w™br ODeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.allocated"trainer/global_step2:JPóí97ršbm JDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed"trainer/global_step2:JP… àao›bj GDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated"trainer/global_step2:JPÚe¢iœbd ADeviceStatsMonitor.on_validation_batch_end/active.small_pool.peak"trainer/global_step2:JPb•¹nbi FDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.current"trainer/global_step2:JP2N€Nkžbf CDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated"trainer/global_step2:JP¥’áBrŸbm JDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated"trainer/global_step2:JPe¥ÒLs bn KDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.current"trainer/global_step2:JP;pQk¡bf CDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed"trainer/global_step2:JPFý¿—q¢bl IDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.peak"trainer/global_step2:JPëí2m£bh EDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated"trainer/global_step2:JPÒzmm¤bh EDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.current"trainer/global_step2:JP"Vïu¥bp MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current"trainer/global_step2:JP;i§Öc¦b^ ;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc"trainer/global_step2:JPÎn‚[§bV 3DeviceStatsMonitor.on_validation_batch_end/num_ooms"trainer/global_step2:JPŠü¹Þk¨bf CDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.peak"trainer/global_step2:JP¿|¨Tp©bk HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated"trainer/global_step2:JPŠS´ïbªb] :DeviceStatsMonitor.on_validation_batch_end/active.all.peak"trainer/global_step2:JP¥†k«bf CDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed"trainer/global_step2:JPß™ó»n¬bi FDeviceStatsMonitor.on_validation_batch_end/oversize_segments.allocated"trainer/global_step2:JPrr­bm JDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current"trainer/global_step2:JP¼=Ìd®b_ DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚22903 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861845973 ]KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚ 1010037550080 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15191 SFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚22773362 OIDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.allocated‚0 ]KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚ 1006320410810 VGDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak‚ 9148669952 gTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚76511834560000 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490911116 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15193 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9954819072 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 % _timestamp‚1.7884458412750194e+09 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 ZMDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.current‚17039360 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 [HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated‚71056120909824 [HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚70045572929024 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚10214180 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863048704 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492427776 SFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚17507256 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 1062387066368 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 ^LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚ 1010529977856 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 1010037550080 YJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚16050716 ^KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.freed‚68214369340416 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 _step‚97 WJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚22788553 VJDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.freed‚5836379 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 OBDeviceStatsMonitor.on_validation_batch_end/active.large_pool.freed‚17507256 cPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.freed‚76511810619904 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 WDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚71055610479104 ^KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚70045572929024 ZNDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚5836534 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 JDDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak‚0 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 H;DeviceStatsMonitor.on_validation_batch_end/active.all.freed‚40280618 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 SFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚17507258 _LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚70045590931968 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚155–WÀ É´R G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15193 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9954819072 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 % _timestamp‚1.7884458412750194e+09 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 ZMDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.current‚17039360 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 [HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated‚71056120909824 [HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚70045572929024 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚10214180 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863048704 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492427776 SFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚17507256 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 1062387066368 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 ^LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚ 1010529977856 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 1010037550080 YJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚16050716 ^KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.freed‚68214369340416 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 _step‚97 WJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚22788553 VJDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.freed‚5836379 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 OBDeviceStatsMonitor.on_validation_batch_end/active.large_pool.freed‚17507256 cPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.freed‚76511810619904 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 WDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚71055610479104 ^KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚70045572929024 ZNDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚5836534 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 JDDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak‚0 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 H;DeviceStatsMonitor.on_validation_batch_end/active.all.freed‚40280618 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 SFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚17507258 _LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚70045590931968 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚155 JDDeviceStatsMonitor.on_validation_batch_end/oversize_segments.current‚0 trainer/global_step‚1208 L?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚40295811 KEDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.freed‚0 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 aODeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.allocated‚ 1006811321926 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414 _runtime‚ 956.747020903 KADeviceStatsMonitor.on_validation_batch_end/active.small_pool.peak‚22630 TFDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.current‚ 510430720 PCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚40295811 WJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚17507258 YKDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.current‚ 135492096 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 XEDeviceStatsMon«" ò.itor.on_validation_batch_end/active_bytes.all.allocated‚71056120909824 MEDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.current‚414 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 RCDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.peak‚ 9643691477 [HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚69221197701702 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚22903 PCDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed‚16050559 LFDeviceStatsMonitor.on_validation_batch_end/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚22903 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861845973 ]KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚ 1010037550080 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15191 SFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚22773362 OIDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.allocated‚0 ]KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚ 1006320410810 VGDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak‚ 9148669952 gTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚76511834560000 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490911116aþ.–¶“Ê‚(:ˆ ”‰æÔð˜”þ gpu.0.gpu‚52.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚156.048' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚31.2096 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0(Ý+QË‚(:Æ ”‰æÔÒÆ»ÿ disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0 disk.sda.out‚ 35.29296875% disk.nvme0n1.out‚2795.86474609375) proc.memory.availableMB‚ 1877.94921875 cpu‚25.35165086954038 proc.cpu.threads‚17 network.recv‚403083' disk.nvme0c0n1.out‚2795.86474609375 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.rbd1.out‚ 5.1083984375% disk./.usageGB‚296.79870986938477 disk.sda3.out‚ 35.29296875 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0* disk./.usagePercent‚19.918410450039126 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.dm-1.in‚0$ memory_percent‚88.53790760040283 network.sent‚ 1.526982e+06 disk.drbd1716.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125# proc.memory.rssMB‚ 2634.93359375 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.sda2.in‚0 disk.dm-1.out‚ 35.29296875* proc.memory.percent‚16.082358360290527" disk.dm-0.out‚2795.86474609375¾·P™HÌbC Purity/1FRt0SRqqt1FBt/FBt_purity"trainer/global_step2:JP¢¼«HÍbC Purity/*FRt0SRqqt4FBt/FBt_purity"trainer/global_step2:JPˆ•—º;Îb6 jet/accuracy_3_of_7"trainer/global_step2:JPýÞßJÏbE "Purity/3FRt1SRqqt2FBt/event_purity"trainer/global_step2:JPÑ{»áNÐbI &Purity/1FRt3SRqqt2FBt/event_proportion"trainer/global_step2:JPÉÔ¶:NÑbI &Purity/1FRt4SRqqt0FBt/event_proportion"trainer/global_step2:JPiû§¾NÒbI &Purity/2FRt1SRqqt0FBt/event_proportion"trainer/global_step2:JP›·ÁOJÓbE "Purity/*FRt2SRqqt3FBt/event_purity"trainer/global_step2:JPðÌç&JÔbE "Purity/0FRt1SRqqt*FBt/event_purity"trainer/global_step2:JPëbèPNÕbI &Purity/0FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JPëïÕNÖbI &Purity/1FRt2SRqqt*FBt/event_proportion"trainer/global_step2:JP±ærÒJ×bE "Purity/2FRt1SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP§ÝãÀJØbE "Purity/2FRt4SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP îˆ;Ùb6 jet/accuracy_2_of_6"trainer/global_step2:JP ¦ÇX@Úb; particle/accuracy_2_of_7"trainer/global_step2:JPÇýF@Ûb; particle/accuracy_3_of_7"trainer/global_step2:JPæh­NÜbI &Purity/0FRt2SRqqt3FBt/event_proportion"trainer/global_step2:JP‹--JÝbE "Purity/0FRt1SRqqt1FBt/event_purity"trainer/global_step2:JP‘}íàNÞbI &Purity/2FRt4SRqqt1FBt/event_proportion"trainer/global_step2:JPçqI"NßbI &Purity/2FRt4SRqqt3FBt/event_proportion"trainer/global_step2:JPQ´ÿ ;àb6 jet/accuracy_2_of_5"trainer/global_step2:JP*P2†HábC Purity/1FRt*SRqqt0FBt/FRt_purity"trainer/global_step2:JP·Ú\JâbE "Purity/3FRt4SRqqt*FBt/event_purity"trainer/global_step2:JPÀŽqJãbE "Purity/*FRt*SRqqt4FBt/SRqqt_purity"trainer/global_step2:JP¯>ÖøJäbE "Purity/*FRt3SRqqt2FBt/event_purity"trainer/global_step2:JP2ÒìéJåbE "Purity/4FRt2SRqqt0FBt/event_purity"trainer/global_step2:JP,ÜðoNæbI &Purity/*FRt*SRqqt1FBt/event_proportion"trainer/global_step2:JPä¶JçbE "Purity/*FRt*SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP½ýJ;èb6 jet/accuracy_4_of_5"trainer/global_step2:JPAzJébE "Purity/0FRt4SRqqt1FBt/event_purity"trainer/global_step2:JP}îxNêbI &Purity/4FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JPÙÒ4/HëbC Purity/*FRt4SRqqt*FBt/FRt_purity"trainer/global_step2:JPC=q™;ìb6 jet/accuracy_5_of_6"trainer/global_step2:JPUEø¬HíbC Purity/*FRt*SRqqt0FBt/FRt_purity"trainer/global_step2:JP§ŽÕ.HîbC Purity/*FRt2SRqqt*FBt/FRt_purity"trainer/global_step2:JP{‰dHïbC Purity/1FRt0SRqqt*FBt/FRt_purity"trainer/global_step2:JPSmÖ%JðbE "Purity/1FRt2SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP\uºQJñbE "Purity/1FRt2SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP“•\ìJòbE "Purity/2FRt2SRqqt0FBt/event_purity"trainer/global_step2:JPtt ½HóbC Purity/4FRt*SRqqt*FBt/FRt_purity"trainer/global_step2:JPjì‚JôbE "Purity/2FRt2SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP{~þNõbI &Purity/*FRt0SRqqt*FBt/event_proportion"trainer/global_step2:JPê`ËjJöbE "Purity/*FRt*SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPBRMH÷bC Purity/*FRt0SRqqt*FBt/FBt_purity"trainer/global_step2:JPÁÿ;øb6 jet/accuracy_1_of_6"trainer/global_step2:JPÿ?ÚÕ@ùb; particle/accuracy_3_of_6"trainer/global_step2:JP2®ÚNúbI &Purity/*FRt*SRqqt*FBt/event_proportion"trainer/global_step2:JPH5 öNûbI &Purity/*FRt2SRqqt3FBt/event_proportion"trainer/global_step2:JPÈ«zoJübE "Purity/0FRt*SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP´úkrJýbE "Purity/1FRt3SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPÃO/JþbE "Purity/2FRt2SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPÛÃ}NÿbI &Purity/4FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JPì,h@€b; particle/accuracy_3_of_4"trainer/global_step2:JP4ù8HbC Purity/3FRt2SRqqt1FBt/FRt_purity"trainer/global_step2:JPèC,WJ‚bE "Purity/*FRt3SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPߥÊ+NƒbI &Purity/0FRt4SRqqt*FBt/event_proportion"trainer/global_step2:JPÅRé]N„bI &Purity/4FRt3SRqqt3FBt/event_proportion"trainer/global_step2:JP¯Èy;…b6 jet/accuracy_6_of_7"trainer/global_step2:JPûu‚xJ†bE "Purity/*FRt1SRqqt4FBt/SRqqt_purity"trainer/global_step2:JP¨8kJ‡bE "Purity/4FRt*SRqqt*FBt/event_purity"trainer/global_step2:JP(8q“JˆbE "Purity/0FRt1SRqqt2FBt/event_purity"trainer/global_step2:JPr­ÎJ‰bE "Purity/0FRt*SRqqt4FBt/event_purity"trainer/global_step2:JPfn¸³NŠbI &Purity/*FRt3SRqqt*FBt/event_proportion"trainer/global_step2:JP}­N‹bI &Purity/0FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JPiDT¦HŒbC Purity/1FRt*SRqqt2FBt/FRt_purity"trainer/global_step2:JPo‘éNbI &Purity/2FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JP:1MáNŽbI &Purity/2FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JP¹6ñNbI &Purity/2FRt4SRqqt2FBt/event_proportion"trainer/global_step2:JPo¬NbI &Purity/3FRt1SRqqt1FBt/event_proportion"trainer/global_step2:JPb ãVN‘bI &Purity/3FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JPd·ðcH’bC Purity/*FRt1SRqqt*FBt/FRt_purity"trainer/global_step2:JP>E•N“bI &Purity/2FRt2SRqqt*FBt/event_proportion"trainer/global_step2:JP(lBöH”bC Purity/3FRt2SRqqt0FBt/FRt_purity"trainer/global_step2:JPC3;•b6 jet/accuracy_3_of_3"trainer/global_step2:JPwMe J–bE "Purity/0FRt4SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPêqzJ—bE "Purity/1FRt2SRqqt2FBt/event_purity"trainer/global_step2:JPŸRd³J˜bE "Purity/2FRt3SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPý·DH™bC Purity/1FRt1SRqqt3FBt/FRt_purity"trainer/global_step2:JP&+ÍrJšbE "Purity/*FRt*SRqqt*FBt/event_purity"trainer/global_step2:JP¡nHN›bI &Purity/*FRt0SRqqt3FBt/event_proportion"trainer/global_step2:JPîvïÐNœbI &Purity/3FRt0SRqqt1FBt/event_proportion"trainer/global_step2:JP)ÿ\‚NbI &Purity/4FRt4SRqqt*FBt/event_proportion"trainer/global_step2:JP~+TJžbE "Purity/*FRt3SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP³éJŸbE "Purity/1FRt*SRqqt4FBt/event_purity"trainer/global_step2:JP¯ÐªÖN bI &Purity/*FRt4SRqqt2FBt/event_proportion"trainer/global_step2:JP©ðJ¡bE "Purity/1FRt0SRqqt3FBt/event_purity"trainer/global_step2:JP "kH¢bC Purity/1FRt1SRqqt*FBt/FBt_purity"trainer/global_step2:JP§³ J£bE "Purity/3FRt2SRqqt*FBt/event_purity"trainer/global_step2:JP^`ÚJ¤bE "Purity/1FRt4SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP ‚¾öJ¥bE "Purity/*FRt*SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPŽeÏnJ¦bE "Purity/0FRt2SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPYŒ×N§bI &Purity/1FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JPÌ?ÈN¨bI &Purity/1FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JPŽ£J©bE "Purity/2FRt2SRqqt1FBt/event_purity"trainer/global_step2:JP‡Œ¶NªbI &Purity/3FRt3SRqqt2FBt/event_proportion"trainer/global_step2:JPÏd>ˆN«bI &Purity/4FRt2SRqqt4FBt/event_proportion"trainer/global_step2:JP+°L)H¬bC Purity/3FRt0SRqqt*FBt/FBt_purity"trainer/global_step2:JPâßqºH­bC Purity/*FRt*SRqqt2FBt/FBt_purity"trainer/global_step2:JPP?9ÖH®bC Purity/0FRt*SRqqt2FBt/FBt_purity"trainer/global_step2:JP™ÛW°J¯bE "Purity/3FRt2SRqqt0FBt/event_purity"trainer/global_step2:JPú¥ÉH°bC Purity/1FRt3SRqqt*FBt/FBt_purity"trainer/global_step2:JP/M>H±bC Purity/0FRt2SRqqt3FBt/FBt_purity"trainer/global_step2:JP.g¾;²b6 loss/val_total_loss"trainer/global_step2:JP;˜PbJ³bE "Purity/3FRt*SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPe œH´bC Purity/*FRt*SRqqt1FBt/FRt_purity"trainer/global_step2:JP1 L.HµbC Purity/*FRt*SRqqt3FBt/FBt_purity"trainer/global_step2:JPš‚}œH¶bC Purity/*FRt0SRqqt3FBt/FRt_purity"trainer/global_step2:JPP5& J·bE "Purity/0FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPVƒ]J¸bE "Purity/0FRt0SRqqt1FBt/event_purity"trainer/global_step2:JPÝANH¹bC Purity/0FRt*SRqqt3FBt/FBt_purity"trainer/global_step2:JP^æ|¢@ºb; particle/accuracy_6_of_6"trainer/global_step2:JP@… H»bC Purity/0FRt2SRqqt2FBt/FBt_purity"trainer/global_step2:JP츮lH¼bC Purity/1FRt2SRqqt1FBt/FBt_purity"trainer/global_step2:JP8ýÇJ½bE "Purity/2FRt1SRqqt2FBt/event_purity"trainer/global_step2:JP?qON¾bI &Purity/3FRt2SRqqt3FBt/event_proportion"trainer/global_step2:JPõÊ]N¿bI &Purity/4FRt3SRqqt0FBt/event_proportion"trainer/global_step2:JPrPnHÀbC Purity/3FRt4SRqqt0FBt/FRt_purity"trainer/global_step2:JPÞl“n;Áb6 jet/accuracy_2_of_7"trainer/global_step2:JPþÂJSJÂbE "Purity/1FRt4SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP>3rÿHÃbC Purity/3FRt3SRqqt*FBt/FRt_purity"trainer/global_step2:JPžÛŸHÄbC Purity/*FRt1SRqqt1FBt/FBt_purity"trainer/global_step2:JP%n‹NÅbI &Purity/0FRt4SRqqt1FBt/event_proportion"trainer/global_step2:JPÿ[™JÆbE "Purity/2FRt*SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPW‹#yHÇbC Purity/2FRt2SRqqt*FBt/FBt_purity"trainer/global_step2:JPIIÚNÈbI &Purity/*FRt*SRqqt0FBt/event_proportion"trainer/global_step2:JPgÈcqÃJ÷bE "Purity/0FRt4SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP‹ABJøbE "Purity/0FRt1SRqqt2FBt/SRqqt_purity"trainer/global_step2:JPž¼žJùbE "Purity/1FRt*SRqqt1FBt/event_purity"trainer/global_step2:JPåIŠŽJúbE "Purity/1FRt*SRqqt3FBt/event_purity"trainer/global_step2:JPÆŸÒFHûbC Purity/2FRt2SRqqt1FBt/FRt_purity"trainer/global_step2:JPž€âBNübI &Purity/4FRt1SRqqt4FBt/event_proportion"trainer/global_step2:JPo1ÈNýbI &Purity/4FRt3SRqqt2FBt/event_proportion"trainer/global_step2:JPÚ€2MNþbI &Purity/4FRt4SRqqt4FBt/event_proportion"trainer/global_step2:JPi©2æ@ÿb; particle/accuracy_1_of_2"trainer/global_step2:JP¨+xH€ bC Purity/1FRt*SRqqt*FBt/FBt_purity"trainer/global_step2:JPŒ¼Q1N bI &Purity/1FRt1SRqqt0FBt/event_proportion"trainer/global_step2:JPþö§+N‚ bI &Purity/2FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JPg)­=@ƒ b; particle/accuracy_2_of_5"trainer/global_step2:JP¼Àî®J„ bE "Purity/3FRt1SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP„NLJ… bE "Purity/3FRt4SRqqt0FBt/event_purity"trainer/global_step2:JP%˜¸ÒJ† bE "Purity/*FRt1SRqqt*FBt/event_purity"trainer/global_step2:JPÑŒJ‡ bE "Purity/0FRt*SRqqt2FBt/event_purity"trainer/global_step2:JP—sçLJˆ bE "Purity/0FRt2SRqqt1FBt/event_purity"trainer/global_step2:JP¿BÓJ‰ bE "Purity/2FRt1SRqqt1FBt/event_purity"trainer/global_step2:JPÂìè@Š b; particle/accuracy_2_of_6"trainer/global_step2:JPRÁOH‹ bC Purity/2FRt2SRqqt2FBt/FBt_purity"trainer/global_step2:JPãÅ~NŒ bI &Purity/1FRt2SRqqt3FBt/event_proportion"trainer/global_step2:JP¼@¡H bC Purity/2FRt*SRqqt*FBt/FRt_purity"trainer/global_step2:JP!dJŽ bE "Purity/2FRt0SRqqt1FBt/event_purity"trainer/global_step2:JPxИÔH bC Purity/2FRt0SRqqt1FBt/FBt_purity"trainer/global_step2:JP \H]H bC Purity/3FRt1SRqqt1FBt/FBt_purity"trainer/global_step2:JP  šßJ‘ bE "Purity/*FRt2SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPöý[ÚN’ bI &Purity/0FRt4SRqqt3FBt/event_proportion"trainer/global_step2:JPéäÖJ“ bE "Purity/2FRt0SRqqt*FBt/event_purity"trainer/global_step2:JPUìo$N” bI &Purity/2FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JPói–dJ• bE "Purity/2FRt2SRqqt2FBt/event_purity"trainer/global_step2:JPxð4¾H– bC Purity/*FRt0SRqqt2FBt/FRt_purity"trainer/global_step2:JPÂÀ N— bI &Purity/1FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JPÜhŠü@˜ b; particle/accuracy_1_of_5"trainer/global_step2:JP<Œß8H™ bC Purity/*FRt*SRqqt4FBt/FBt_purity"trainer/global_step2:JPŽl—THš bC Purity/0FRt*SRqqt4FBt/FBt_purity"trainer/global_step2:JP…dv²J› bE "Purity/1FRt1SRqqt3FBt/SRqqt_purity"trainer/global_step2:JPÒ~UJœ bE "Purity/*FRt*SRqqt2FBt/event_purity"trainer/global_step2:JP½PCJ bE "Purity/*FRt1SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPÌâªýJž bE "Purity/3FRt*SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP -§”HŸ bC Purity/0FRt4SRqqt*FBt/FBt_purity"trainer/global_step2:JPB×3–H  bC Purity/2FRt4SRqqt*FBt/FRt_purity"trainer/global_step2:JP?g@‚N¡ bI &Purity/*FRt3SRqqt1FBt/event_proportion"trainer/global_step2:JPò±=N¢ bI &Purity/0FRt2SRqqt*FBt/event_proportion"trainer/global_step2:JPU½;$H£ bC Purity/2FRt1SRqqt*FBt/FRt_purity"trainer/global_step2:JP:'rN¤ bI &Purity/3FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JPÊî¦N¥ bI &Purity/4FRt0SRqqt1FBt/event_proportion"trainer/global_step2:JPÍ%QÝJ¦ bE "Purity/0FRt3SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPÑrƒ@§ b; particle/accuracy_6_of_7"trainer/global_step2:JP°ÓÇJ¨ bE "Purity/2FRt*SRqqt3FBt/event_purity"trainer/global_step2:JPù6?»H© bC Purity/1FRt1SRqqt2FBt/FBt_purity"trainer/global_step2:JPÀPZiJª bE "Purity/3FRt*SRqqt*FBt/event_purity"trainer/global_step2:JPüyÍœN« bI &Purity/3FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JP"»¯N¬ bI &Purity/4FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JPK÷ñH­ bC Purity/3FRt0SRqqt2FBt/FBt_purity"trainer/global_step2:JPCI8hJ® bE "Purity/*FRt3SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP¶-ðDJ¯ bE "Purity/2FRt*SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP5ĺXN° bI &Purity/4FRt4SRqqt2FBt/event_proportion"trainer/global_step2:JPlä|ÓH± bC Purity/3FRt0SRqqt1FBt/FRt_purity"trainer/global_step2:JPŸ(AÚJ² bE "Purity/0FRt1SRqqt3FBt/event_purity"trainer/global_step2:JPè@2 J³ bE "Purity/0FRt2SRqqt3FBt/event_purity"trainer/global_step2:JPèWJ¬J´ bE "Purity/1FRt*SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPÞ%R™Hµ bC Purity/1FRt1SRqqt1FBt/FRt_purity"trainer/global_step2:JP}ö7N¶ bI &Purity/3FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JPçO§7J· bE "Purity/*FRt4SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP®ò;¸ b6 jet/accuracy_3_of_6"trainer/global_step2:JP1 õíH¹ bC Purity/1FRt*SRqqt4FBt/FBt_purity"trainer/global_step2:JPVoBýHº bC Purity/1FRt1SRqqt3FBt/FBt_purity"trainer/global_step2:JP•gßH» bC Purity/0FRt1SRqqt1FBt/FBt_purity"trainer/global_step2:JP2E`‹J¼ bE "Purity/1FRt3SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP ˜Ï'H½ bC Purity/2FRt*SRqqt1FBt/FBt_purity"trainer/global_step2:JP&….ˆH¾ bC Purity/2FRt0SRqqt*FBt/FRt_purity"trainer/global_step2:JP_:EN¿ bI &Purity/2FRt3SRqqt3FBt/event_proportion"trainer/global_step2:JPÍŠ[µNÀ bI &Purity/3FRt3SRqqt0FBt/event_proportion"trainer/global_step2:JP¨ã]ÊHÁ bC Purity/3FRt1SRqqt*FBt/FRt_purity"trainer/global_step2:JP¯ð©èN bI &Purity/0FRt1SRqqt1FBt/event_proportion"trainer/global_step2:JP8ÛÎkHà bC Purity/0FRt2SRqqt*FBt/FBt_purity"trainer/global_step2:JPÃJÄ bE "Purity/1FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPe„" JÅ bE "Purity/2FRt*SRqqt*FBt/event_purity"trainer/global_step2:JPHáMNÆ bI &Purity/2FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JPq:2VNÇ bI &Purity/3FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JP9’n•NÈ bI &Purity/4FRt0SRqqt4FBt/event_proportion"trainer/global_step2:JP`Ó3&HÉ bC Purity/1FRt3SRqqt1FBt/FRt_purity"trainer/global_step2:JPÙÚÚ^JÊ bE "Purity/*FRt0SRqqt*FBt/event_purity"trainer/global_step2:JP¯ôÚåHË bC Purity/*FRt0SRqqt0FBt/FRt_purity"trainer/global_step2:JPð««pNÌ bI &Purity/0FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JPç.ƒJÍ bE "Purity/1FRt0SRqqt1FBt/event_purity"trainer/global_step2:JP˜¾²«NÎ bI &Purity/3FRt1SRqqt0FBt/event_proportion"trainer/global_step2:JPäÕJÏ bE "Purity/1FRt3SRqqt1FBt/event_purity"trainer/global_step2:JPé©CJÐ bE "Purity/3FRt1SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP*cæ~JÑ bE "Purity/4FRt1SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP whHÒ bC Purity/0FRt*SRqqt*FBt/FBt_purity"trainer/global_step2:JP)X=kHÓ bC Purity/*FRt*SRqqt*FBt/FBt_purity"trainer/global_step2:JP‚ÞœÐHÔ bC Purity/0FRt0SRqqt1FBt/FBt_purity"trainer/global_step2:JPGÀ^HÕ bC Purity/1FRt3SRqqt2FBt/FBt_purity"trainer/global_step2:JP¼Õ? HÖ bC Purity/*FRt0SRqqt2FBt/FBt_purity"trainer/global_step2:JPßTfeN× bI &Purity/0FRt4SRqqt2FBt/event_proportion"trainer/global_step2:JP»$ù¨NØ bI &Purity/1FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JP¬ç;JÙ bE "Purity/2FRt3SRqqt0FBt/event_purity"trainer/global_step2:JP ^o-NÚ bI &Purity/4FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JPÒ`ï¤JÛ bE "Purity/*FRt4SRqqt0FBt/event_purity"trainer/global_step2:JPjŠ\1HÜ bC Purity/1FRt*SRqqt4FBt/FRt_purity"trainer/global_step2:JP”˜p HÝ bC Purity/1FRt3SRqqt0FBt/FRt_purity"trainer/global_step2:JP©0HÞ bC Purity/4FRt0SRqqt0FBt/FRt_purity"trainer/global_step2:JP_ãƧHß bC Purity/*FRt*SRqqt3FBt/FRt_purity"trainer/global_step2:JPZ3µLHà bC Purity/*FRt0SRqqt1FBt/FRt_purity"trainer/global_step2:JPZ'ƒÊNá bI &Purity/*FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JP8âKHâ bC Purity/*FRt0SRqqt3FBt/FBt_purity"trainer/global_step2:JPIRƒNã bI &Purity/3FRt*SRqqt*FBt/event_proportion"trainer/global_step2:JPãÙPÍJä bE "Purity/0FRt0SRqqt3FBt/event_purity"trainer/global_step2:JPðöÁŠNå bI &Purity/4FRt2SRqqt*FBt/event_proportion"trainer/global_step2:JPÞTÈJæ bE "Purity/*FRt0SRqqt2FBt/event_purity"trainer/global_step2:JPu8¡¨Nç bI &Purity/2FRt*SRqqt1FBt/event_proportion"trainer/global_step2:JPiœDNè bI &Purity/2FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JP„5˜pNé bI &Purity/3FRt*SRqqt4FBt/event_proportion"trainer/global_step2:JPíiÒNê bI &Purity/3FRt2SRqqt4FBt/event_proportion"trainer/global_step2:JPˆV#w;ë b6 jet/accuracy_1_of_3"trainer/global_step2:JP @ì b; particle/accuracy_3_of_3"trainer/global_step2:JPEãQJí bE "Purity/*FRt1SRqqt0FBt/event_purity"trainer/global_step2:JPÞ”ŸJî bE "Purity/1FRt1SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP·G{Jï bE "Purity/1FRt2SRqqt0FBt/event_purity"trainer/global_step2:JPüšrƒHð bC Purity/3FRt*SRqqt1FBt/FBt_purity"trainer/global_step2:JPk)ò@ñ b; particle/accuracy_1_of_6"trainer/global_step2:JP¦c¨Jò bE "Purity/*FRt0SRqqt0FBt/event_purity"trainer/global_step2:JP=é@ó b; particle/accuracy_1_of_3"trainer/global_step2:JP‰¯ÐWHô bC Purity/3FRt2SRqqt*FBt/FBt_purity"trainer/global_step2:JP¶Hõ bC Purity/2FRt2SRqqt2FBt/FRt_purity"trainer/global_step2:JPAVÎHö bC Purity/3FRt0SRqqt0FBt/FRt_purity"trainer/global_step2:JP@o•ÇJ÷ bE "Purity/*FRt0SRqqt3FBt/event_purity"trainer/global_step2:JP…-CÕNø bI &Purity/*FRt1SRqqt4FBt/event_proportion"trainer/global_step2:JPOç©Nù bI &Purity/*FRt2SRqqt4FBt/event_proportion"trainer/global_step2:JPö›%Hú bC Purity/1FRt1SRqqt0FBt/FRt_purity"trainer/global_step2:JP˜ýÚAHû bC Purity/*FRt3SRqqt1FBt/FBt_purity"trainer/global_step2:JPaÏ,Jü bE "Purity/2FRt0SRqqt3FBt/event_purity"trainer/global_step2:JP(€ žNý bI &Purity/2FRt1SRqqt1FBt/event_proportion"trainer/global_step2:JP§õ¦ÿJþ bE "Purity/2FRt1SRqqt2FBt/SRqqt_purity"trainer/global_step2:JPЄòÎHÿ bC Purity/2FRt3SRqqt*FBt/FBt_purity"trainer/global_step2:JPÙ]öJ€ bE "Purity/*FRt1SRqqt3FBt/event_purity"trainer/global_step2:JP­d*J bE "Purity/1FRt1SRqqt4FBt/SRqqt_purity"trainer/global_step2:JP#ûðîN‚ bI &Purity/*FRt*SRqqt4FBt/event_proportion"trainer/global_step2:JPÂmá^Jƒ bE "Purity/*FRt3SRqqt*FBt/event_purity"trainer/global_step2:JPƒo74J„ bE "Purity/2FRt1SRqqt*FBt/event_purity"trainer/global_step2:JPTƒ%ÉN… bI &Purity/4FRt0SRqqt*FBt/event_proportion"trainer/global_step2:JP êWì;† b6 jet/accuracy_1_of_5"trainer/global_step2:JP`ø·_J‡ bE "Purity/0FRt*SRqqt3FBt/SRqqt_purity"trainer/global_step2:JPûH° bC Purity/1FRt*SRqqt3FBt/FRt_purity"trainer/global_step2:JPêÊ4²J± bE "Purity/*FRt0SRqqt4FBt/event_purity"trainer/global_step2:JPØ÷‡åJ² bE "Purity/1FRt2SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP?uEú;³ b6 jet/accuracy_7_of_7"trainer/global_step2:JPˆÄ$SJ´ bE "Purity/1FRt0SRqqt4FBt/event_purity"trainer/global_step2:JPpZºNµ bI &Purity/1FRt4SRqqt4FBt/event_proportion"trainer/global_step2:JPWÈBvN¶ bI &Purity/2FRt3SRqqt0FBt/event_proportion"trainer/global_step2:JP’̲¬J· bE "Purity/3FRt4SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP±­rJ¸ bE "Purity/*FRt1SRqqt1FBt/event_purity"trainer/global_step2:JP–ÛÂN¹ bI &Purity/*FRt4SRqqt0FBt/event_proportion"trainer/global_step2:JP ÜÏJº bE "Purity/0FRt1SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP`zP±N» bI &Purity/0FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JPjÏMN¼ bI &Purity/0FRt4SRqqt4FBt/event_proportion"trainer/global_step2:JPï=ßN½ bI &Purity/1FRt3SRqqt*FBt/event_proportion"trainer/global_step2:JPпøN¾ bI &Purity/3FRt1SRqqt4FBt/event_proportion"trainer/global_step2:JPûôò€N¿ bI &Purity/3FRt3SRqqt1FBt/event_proportion"trainer/global_step2:JP>žá‰JÀ bE "Purity/0FRt*SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPÝÜQKHÁ bC Purity/2FRt1SRqqt0FBt/FRt_purity"trainer/global_step2:JPÁ¢‚=@ b; particle/accuracy_4_of_5"trainer/global_step2:JP´Ã¸#@à b; particle/accuracy_5_of_5"trainer/global_step2:JPžÚBEHÄ bC Purity/*FRt1SRqqt3FBt/FRt_purity"trainer/global_step2:JPýÚèHÅ bC Purity/*FRt2SRqqt*FBt/FBt_purity"trainer/global_step2:JP ià•NÆ bI &Purity/0FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JP©—õmJÇ bE "Purity/1FRt*SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPB¨HÈ bC Purity/1FRt0SRqqt*FBt/FBt_purity"trainer/global_step2:JPú/ÓÅNÉ bI &Purity/1FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JPîY…ZNÊ bI &Purity/2FRt2SRqqt3FBt/event_proportion"trainer/global_step2:JP$[uù;Ë b6 jet/accuracy_1_of_1"trainer/global_step2:JPr2â?JÌ bE "Purity/*FRt2SRqqt3FBt/SRqqt_purity"trainer/global_step2:JPúŸHÍ bC Purity/*FRt0SRqqt*FBt/FRt_purity"trainer/global_step2:JPÀ.Þ"JÎ bE "Purity/*FRt2SRqqt0FBt/event_purity"trainer/global_step2:JPïnw JÏ bE "Purity/0FRt*SRqqt1FBt/event_purity"trainer/global_step2:JP¸xÇDNÐ bI &Purity/0FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JPú´¡ïNÑ bI &Purity/*FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JP²h«JÒ bE "Purity/0FRt2SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP’dÅENÓ bI &Purity/1FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JPs…ÞžJÔ bE "Purity/2FRt1SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP ò™ˆJÕ bE "Purity/4FRt*SRqqt0FBt/event_purity"trainer/global_step2:JP¯¶»CNÖ bI &Purity/*FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JPñ¶^ZJ× bE "Purity/1FRt1SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPˆÏVHØ bC Purity/2FRt*SRqqt2FBt/FRt_purity"trainer/global_step2:JPdMxãHÙ bC Purity/2FRt0SRqqt0FBt/FRt_purity"trainer/global_step2:JPÌüº„NÚ bI &Purity/3FRt*SRqqt0FBt/event_proportion"trainer/global_step2:JP€ó@nNÛ bI &Purity/4FRt3SRqqt1FBt/event_proportion"trainer/global_step2:JP‹B&¬HÜ bC Purity/3FRt1SRqqt0FBt/FRt_purity"trainer/global_step2:JPFADHÝ bC Purity/*FRt4SRqqt*FBt/FBt_purity"trainer/global_step2:JPÙŒì3NÞ bI &Purity/*FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JPËL nNß bI &Purity/*FRt4SRqqt1FBt/event_proportion"trainer/global_step2:JPmÚSÌJà bE "Purity/2FRt3SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPÕ_ÈHá bC Purity/*FRt3SRqqt1FBt/FRt_purity"trainer/global_step2:JP¤[SJâ bE "Purity/*FRt4SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP¯D„Jã bE "Purity/4FRt0SRqqt0FBt/event_purity"trainer/global_step2:JPÃð¹zNä bI &Purity/2FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JPqBgNå bI &Purity/3FRt3SRqqt4FBt/event_proportion"trainer/global_step2:JPé“Ñ{<æ b7 particle/specificity"trainer/global_step2:JP³÷þJç bE "Purity/0FRt4SRqqt*FBt/event_purity"trainer/global_step2:JP*߆‡Hè bC Purity/3FRt*SRqqt2FBt/FBt_purity"trainer/global_step2:JP 0ÜÆNé bI &Purity/0FRt0SRqqt4FBt/event_proportion"trainer/global_step2:JPÞ]ïOJê bE "Purity/2FRt2SRqqt*FBt/event_purity"trainer/global_step2:JPô€A<;ë b6 jet/accuracy_3_of_4"trainer/global_step2:JPBkIHì bC Purity/*FRt3SRqqt2FBt/FBt_purity"trainer/global_step2:JPÅ>2 Ní bI &Purity/2FRt0SRqqt*FBt/event_proportion"trainer/global_step2:JPnYézHî bC Purity/2FRt1SRqqt2FBt/FRt_purity"trainer/global_step2:JPªf .Nï bI &Purity/3FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JPå¢[°Jð bE "Purity/1FRt3SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPÇeסHñ bC Purity/3FRt*SRqqt1FBt/FRt_purity"trainer/global_step2:JPÖVp£Jò bE "Purity/4FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPyšqNó bI &Purity/*FRt0SRqqt1FBt/event_proportion"trainer/global_step2:JPÖÎœJô bE "Purity/*FRt1SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP:[†ÎJõ bE "Purity/1FRt2SRqqt*FBt/event_purity"trainer/global_step2:JP$jÅtJö bE "Purity/1FRt2SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPjžöN÷ bI &Purity/4FRt4SRqqt3FBt/event_proportion"trainer/global_step2:JPÒ‡‹1@ø b; particle/accuracy_2_of_4"trainer/global_step2:JPâJù bE "Purity/3FRt0SRqqt*FBt/event_purity"trainer/global_step2:JP(KHú bC Purity/2FRt4SRqqt0FBt/FRt_purity"trainer/global_step2:JPÿOž¨Hû bC Purity/*FRt0SRqqt1FBt/FBt_purity"trainer/global_step2:JP‘ „Jü bE "Purity/0FRt0SRqqt0FBt/event_purity"trainer/global_step2:JP¸V[AHý bC Purity/1FRt0SRqqt4FBt/FBt_purity"trainer/global_step2:JPAyGuHþ bC Purity/*FRt1SRqqt4FBt/FRt_purity"trainer/global_step2:JP×ÏNÿ bI &Purity/*FRt*SRqqt3FBt/event_proportion"trainer/global_step2:JP‚l1?N€ bI &Purity/0FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JP‚ÓJ(N bI &Purity/1FRt0SRqqt*FBt/event_proportion"trainer/global_step2:JP|]mÔJ‚ bE "Purity/*FRt4SRqqt1FBt/event_purity"trainer/global_step2:JP”ÉxJƒ bE "Purity/1FRt*SRqqt4FBt/SRqqt_purity"trainer/global_step2:JPÜ-¯ŽH„ bC Purity/2FRt*SRqqt0FBt/FRt_purity"trainer/global_step2:JP© 5pN… bI &Purity/2FRt*SRqqt3FBt/event_proportion"trainer/global_step2:JPIÊòŽH† bC Purity/2FRt0SRqqt2FBt/FRt_purity"trainer/global_step2:JP¦Ó¤€N‡ bI &Purity/3FRt3SRqqt*FBt/event_proportion"trainer/global_step2:JPÚHìHˆ bC Purity/3FRt1SRqqt2FBt/FRt_purity"trainer/global_step2:JPxodQJ‰ bE "Purity/0FRt*SRqqt*FBt/event_purity"trainer/global_step2:JP¾rµ'NŠ bI &Purity/0FRt*SRqqt1FBt/event_proportion"trainer/global_step2:JPAKÈÔN‹ bI &Purity/0FRt3SRqqt3FBt/event_proportion"trainer/global_step2:JPÃÒ‡:NŒ bI &Purity/2FRt3SRqqt1FBt/event_proportion"trainer/global_step2:JP¡Ý»çH bC Purity/0FRt0SRqqt3FBt/FBt_purity"trainer/global_step2:JP?JŽ bE "Purity/0FRt3SRqqt0FBt/event_purity"trainer/global_step2:JP{®’áN bI &Purity/0FRt0SRqqt3FBt/event_proportion"trainer/global_step2:JP vH bC Purity/0FRt1SRqqt2FBt/FBt_purity"trainer/global_step2:JPGíSN‘ bI &Purity/0FRt2SRqqt4FBt/event_proportion"trainer/global_step2:JP5æ6 H’ bC Purity/1FRt2SRqqt*FBt/FBt_purity"trainer/global_step2:JP"”§EN“ bI &Purity/1FRt4SRqqt*FBt/event_proportion"trainer/global_step2:JPM (H” bC Purity/2FRt*SRqqt2FBt/FBt_purity"trainer/global_step2:JPˆ>tT;• b6 jet/accuracy_2_of_3"trainer/global_step2:JPÏ‘ÞÛJ– bE "Purity/*FRt2SRqqt2FBt/event_purity"trainer/global_step2:JP㲄H— bC Purity/0FRt0SRqqt2FBt/FBt_purity"trainer/global_step2:JPpÓ“J˜ bE "Purity/2FRt*SRqqt0FBt/event_purity"trainer/global_step2:JPè!0»;™ b6 jet/accuracy_2_of_2"trainer/global_step2:JP38qHš bC Purity/1FRt3SRqqt1FBt/FBt_purity"trainer/global_step2:JPLŸ¡H› bC Purity/1FRt4SRqqt*FBt/FRt_purity"trainer/global_step2:JP;ºj´Jœ bE "Purity/*FRt1SRqqt4FBt/event_purity"trainer/global_step2:JPÀ`H bC Purity/4FRt1SRqqt0FBt/FRt_purity"trainer/global_step2:JPGÖìJJž bE "Purity/1FRt*SRqqt0FBt/event_purity"trainer/global_step2:JPvœ©¯HŸ bC Purity/2FRt*SRqqt1FBt/FRt_purity"trainer/global_step2:JP講N  bI &Purity/4FRt*SRqqt0FBt/event_proportion"trainer/global_step2:JPI~é™H¡ bC Purity/2FRt*SRqqt3FBt/FBt_purity"trainer/global_step2:JP(ŬH¢ bC Purity/0FRt1SRqqt3FBt/FBt_purity"trainer/global_step2:JPY| ùH£ bC Purity/*FRt3SRqqt0FBt/FRt_purity"trainer/global_step2:JP„2f,H¤ bC Purity/1FRt1SRqqt1FBt/FBt_purity"trainer/global_step2:JPlÇJ¥ bE "Purity/2FRt0SRqqt0FBt/event_purity"trainer/global_step2:JPS_7ÄJ¦ bE "Purity/3FRt2SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPÞƱ;§ b6 jet/accuracy_5_of_5"trainer/global_step2:JP§'VJ¨ bE "Purity/3FRt*SRqqt2FBt/event_purity"trainer/global_step2:JPÞÅ8ÀJ© bE "Purity/1FRt4SRqqt*FBt/event_purity"trainer/global_step2:JPû4t‡Jª bE "Purity/1FRt3SRqqt2FBt/event_purity"trainer/global_step2:JPBÄ©:@« b; particle/accuracy_3_of_5"trainer/global_step2:JPl6ßH¬ bC Purity/3FRt0SRqqt1FBt/FBt_purity"trainer/global_step2:JP.Ä’J­ bE "Purity/4FRt1SRqqt0FBt/event_purity"trainer/global_step2:JP~ /×N® bI &Purity/*FRt3SRqqt3FBt/event_proportion"trainer/global_step2:JP~¨g-H¯ bC Purity/2FRt2SRqqt*FBt/FRt_purity"trainer/global_step2:JPYŸ›¶N° bI &Purity/4FRt*SRqqt3FBt/event_proportion"trainer/global_step2:JP0ÔH?H± bC Purity/1FRt0SRqqt4FBt/FRt_purity"trainer/global_step2:JPËfUÅH² bC Purity/4FRt2SRqqt0FBt/FRt_purity"trainer/global_step2:JPNEùJ³ bE "Purity/*FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPÁ5dN´ bI &Purity/0FRt3SRqqt1FBt/event_proportion"trainer/global_step2:JPøŠ8–Hµ bC Purity/3FRt*SRqqt0FBt/FRt_purity"trainer/global_step2:JPH rlN¶ bI &Purity/3FRt4SRqqt3FBt/event_proportion"trainer/global_step2:JPRhH· bC Purity/*FRt1SRqqt4FBt/FBt_purity"trainer/global_step2:JP]+¹yJ¸ bE "Purity/*FRt2SRqqt*FBt/event_purity"trainer/global_step2:JP_”pËJ¹ bE "Purity/1FRt*SRqqt2FBt/event_purity"trainer/global_step2:JPÑ=Ï«Jº bE "Purity/2FRt3SRqqt*FBt/event_purity"trainer/global_step2:JPà™g<» b7 particle/sensitivity"trainer/global_step2:JP™Â$„J¼ bE "Purity/*FRt*SRqqt4FBt/event_purity"trainer/global_step2:JP²ž•J½ bE "Purity/0FRt4SRqqt0FBt/event_purity"trainer/global_step2:JP]ó J¾ bE "Purity/4FRt2SRqqt*FBt/event_purity"trainer/global_step2:JP#g…_J¿ bE "Purity/*FRt1SRqqt2FBt/SRqqt_purity"trainer/global_step2:JPâ=ÚNÀ bI &Purity/*FRt4SRqqt3FBt/event_proportion"trainer/global_step2:JPGYÊNÁ bI &Purity/0FRt*SRqqt3FBt/event_proportion"trainer/global_step2:JP¨ 2?H bC Purity/1FRt0SRqqt3FBt/FRt_purity"trainer/global_step2:JPGFÅHà bC Purity/2FRt0SRqqt2FBt/FBt_purity"trainer/global_step2:JPálUNÄ bI &Purity/2FRt4SRqqt0FBt/event_proportion"trainer/global_step2:JP#é’NÅ bI &Purity/2FRt4SRqqt4FBt/event_proportion"trainer/global_step2:JPÞL †NÆ bI &Purity/4FRt3SRqqt4FBt/event_proportion"trainer/global_step2:JPÍUÉ JÇ bE "Purity/1FRt1SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPÙéSHÈ bC Purity/1FRt3SRqqt*FBt/FRt_purity"trainer/global_step2:JP¶J5JÉ bE "Purity/2FRt0SRqqt2FBt/event_purity"trainer/global_step2:JP!ºTNÊ bI &Purity/2FRt1SRqqt4FBt/event_proportion"trainer/global_step2:JPépÊNË bI &Purity/3FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JPaÐ@Ì b; particle/accuracy_1_of_4"trainer/global_step2:JP÷ž–XJÍ bE "Purity/3FRt1SRqqt0FBt/event_purity"trainer/global_step2:JPfí”JÎ bE "Purity/0FRt2SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP™SáNÏ bI &Purity/0FRt1SRqqt4FBt/event_proportion"trainer/global_step2:JPu›bSNÐ bI &Purity/1FRt1SRqqt3FBt/event_proportion"trainer/global_step2:JPÖqÔàHÑ bC Purity/2FRt0SRqqt1FBt/FRt_purity"trainer/global_step2:JPŠ²HÒ bC Purity/3FRt1SRqqt1FBt/FRt_purity"trainer/global_step2:JPûƒ¿JÓ bE "Purity/3FRt4SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPŽT¾`HÔ bC Purity/0FRt3SRqqt1FBt/FBt_purity"trainer/global_step2:JP"¶„VHÕ bC Purity/2FRt0SRqqt3FBt/FBt_purity"trainer/global_step2:JPç*JÖ bE "Purity/4FRt1SRqqt*FBt/event_purity"trainer/global_step2:JP¸‹AN× bI &Purity/*FRt3SRqqt0FBt/event_proportion"trainer/global_step2:JP8iaJØ bE "Purity/0FRt2SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPW´“DHÙ bC Purity/1FRt1SRqqt*FBt/FRt_purity"trainer/global_step2:JP…,HÚ bC Purity/3FRt1SRqqt2FBt/FBt_purity"trainer/global_step2:JP/®«©JÛ bE "Purity/*FRt*SRqqt3FBt/SRqqt_purity"trainer/global_step2:JPJeR‹HÜ bC Purity/*FRt3SRqqt2FBt/FRt_purity"trainer/global_step2:JPMë HÝ bC Purity/2FRt1SRqqt2FBt/FBt_purity"trainer/global_step2:JPéFÆNÞ bI &Purity/3FRt1SRqqt3FBt/event_proportion"trainer/global_step2:JPtοaHß bC Purity/3FRt0SRqqt*FBt/FRt_purity"trainer/global_step2:JPÍj·ÊHà bC Purity/4FRt0SRqqt*FBt/FRt_purity"trainer/global_step2:JP;É+Há bC Purity/*FRt1SRqqt*FBt/FBt_purity"trainer/global_step2:JPòXûÍHâ bC Purity/2FRt2SRqqt1FBt/FBt_purity"trainer/global_step2:JPv›ÔNã bI &Purity/*FRt3SRqqt4FBt/event_proportion"trainer/global_step2:JPõ$mHä bC Purity/2FRt1SRqqt1FBt/FRt_purity"trainer/global_step2:JP!~»Jå bE "Purity/3FRt*SRqqt0FBt/event_purity"trainer/global_step2:JP¨YìÁNæ bI &Purity/4FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JPqt“ÁHç bC Purity/3FRt*SRqqt2FBt/FRt_purity"trainer/global_step2:JPæ¶ð@è b; particle/accuracy_4_of_4"trainer/global_step2:JPhÆBHé bC Purity/0FRt0SRqqt4FBt/FBt_purity"trainer/global_step2:JP#ìÿHê bC Purity/*FRt*SRqqt*FBt/FRt_purity"trainer/global_step2:JP3éÊéNë bI &Purity/*FRt0SRqqt4FBt/event_proportion"trainer/global_step2:JPcýyHì bC Purity/*FRt2SRqqt0FBt/FRt_purity"trainer/global_step2:JP\_‚ÑNí bI &Purity/1FRt*SRqqt*FBt/event_proportion"trainer/global_step2:JP¾†Hî bC Purity/1FRt0SRqqt0FBt/FRt_purity"trainer/global_step2:JPåˆ@ï b; particle/accuracy_1_of_1"trainer/global_step2:JPûÙ]üHð bC Purity/0FRt4SRqqt1FBt/FBt_purity"trainer/global_step2:JPÏ1RJñ bE "Purity/1FRt1SRqqt3FBt/event_purity"trainer/global_step2:JP ¶Nò bI &Purity/*FRt4SRqqt*FBt/event_proportion"trainer/global_step2:JPf]ŠNó bI &Purity/1FRt0SRqqt3FBt/event_proportion"trainer/global_step2:JP)=o¶Jô bE "Purity/1FRt1SRqqt0FBt/event_purity"trainer/global_step2:JPH]©vHõ bC Purity/2FRt0SRqqt*FBt/FBt_purity"trainer/global_step2:JPžðˆHö bC Purity/3FRt1SRqqt*FBt/FBt_purity"trainer/global_step2:JPÙË—`H÷ bC Purity/*FRt4SRqqt0FBt/FRt_purity"trainer/global_step2:JP×i¼Ó;ø b6 jet/accuracy_4_of_7"trainer/global_step2:JPÍq_Hù bC Purity/4FRt*SRqqt0FBt/FRt_purity"trainer/global_step2:JPyæJú bE "Purity/0FRt1SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPpxpvHû bC Purity/1FRt*SRqqt1FBt/FBt_purity"trainer/global_step2:JP¬ø5Jü bE "Purity/3FRt3SRqqt0FBt/event_purity"trainer/global_step2:JPÈel”Jý bE "Purity/*FRt1SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPŒB³²Hþ bC Purity/*FRt1SRqqt2FBt/FBt_purity"trainer/global_step2:JP5k4ZNÿ bI &Purity/2FRt3SRqqt*FBt/event_proportion"trainer/global_step2:JP‘ ôN€ bI &Purity/4FRt1SRqqt0FBt/event_proportion"trainer/global_step2:JPyùH bC Purity/2FRt1SRqqt*FBt/FBt_purity"trainer/global_step2:JP¡³¢YJ‚ bE "Purity/1FRt*SRqqt3FBt/SRqqt_purity"trainer/global_step2:JPìÊÔHƒ bC Purity/*FRt1SRqqt1FBt/FRt_purity"trainer/global_step2:JPU>Ý;„ b6 jet/accuracy_2_of_4"trainer/global_step2:JP!µä);… b6 jet/accuracy_1_of_7"trainer/global_step2:JPðÓn¸H† bC Purity/*FRt1SRqqt3FBt/FBt_purity"trainer/global_step2:JP Í[J‡ bE "Purity/1FRt1SRqqt4FBt/event_purity"trainer/global_step2:JP ]pÂNˆ bI &Purity/0FRt0SRqqt*FBt/event_proportion"trainer/global_step2:JPSóžßN‰ bI &Purity/0FRt1SRqqt3FBt/event_proportion"trainer/global_step2:JPƒŒøHŠ bC Purity/1FRt2SRqqt1FBt/FRt_purity"trainer/global_step2:JP±÷vÖN‹ bI &Purity/4FRt1SRqqt3FBt/event_proportion"trainer/global_step2:JPTª®áJŒ bE "Purity/3FRt0SRqqt1FBt/event_purity"trainer/global_step2:JP>«ÔH bC Purity/*FRt*SRqqt1FBt/FBt_purity"trainer/global_step2:JPŒKœhHŽ bC Purity/0FRt*SRqqt1FBt/FBt_purity"trainer/global_step2:JP_„I»H bC Purity/0FRt0SRqqt*FBt/FBt_purity"trainer/global_step2:JPi{H bC Purity/1FRt0SRqqt2FBt/FRt_purity"trainer/global_step2:JPÉnÎ1H‘ bC Purity/*FRt2SRqqt2FBt/FRt_purity"trainer/global_step2:JP,¼\H’ bC Purity/0FRt1SRqqt*FBt/FBt_purity"trainer/global_step2:JPGR J“ bE "Purity/1FRt3SRqqt*FBt/event_purity"trainer/global_step2:JP³CL&H” bC Purity/2FRt*SRqqt*FBt/FBt_purity"trainer/global_step2:JPÝúIoH• bC Purity/1FRt2SRqqt2FBt/FBt_purity"trainer/global_step2:JP"£±–J– bE "Purity/2FRt3SRqqt1FBt/event_purity"trainer/global_step2:JP…ë[íJ— bE "Purity/*FRt*SRqqt0FBt/event_purity"trainer/global_step2:JPȉ_ÏJ˜ bE "Purity/0FRt2SRqqt0FBt/event_purity"trainer/global_step2:JP _"ÚH™ bC Purity/0FRt2SRqqt1FBt/FBt_purity"trainer/global_step2:JPcA|ØJš bE "Purity/1FRt*SRqqt*FBt/event_purity"trainer/global_step2:JPBÕJ› bE "Purity/2FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPþuˆNœ bI &Purity/2FRt3SRqqt4FBt/event_proportion"trainer/global_step2:JP%qÂéN bI &Purity/4FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JPüiJâHž bC Purity/1FRt*SRqqt*FBt/FRt_purity"trainer/global_step2:JPµÎoGNŸ bI &Purity/1FRt4SRqqt2FBt/event_proportion"trainer/global_step2:JP*vNbJ  bE "Purity/3FRt2SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP¢ÒuN¡ bI &Purity/1FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JP¤,íJ¢ bE "Purity/0FRt4SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPh¨CS@£ b; particle/accuracy_5_of_7"trainer/global_step2:JPk‡þJ¤ bE "Purity/0FRt1SRqqt3FBt/SRqqt_purity"trainer/global_step2:JPïv/J¥ bE "Purity/0FRt2SRqqt3FBt/SRqqt_purity"trainer/global_step2:JP³ãJ¦ bE "Purity/*FRt*SRqqt1FBt/event_purity"trainer/global_step2:JPL§eN§ bI &Purity/2FRt3SRqqt2FBt/event_proportion"trainer/global_step2:JPà—ô{N¨ bI &Purity/*FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JPA† N© bI &Purity/0FRt3SRqqt0FBt/event_proportion"trainer/global_step2:JP½1Jª bE "Purity/1FRt2SRqqt1FBt/event_purity"trainer/global_step2:JPýrûJ« bE "Purity/2FRt*SRqqt2FBt/event_purity"trainer/global_step2:JP "1J¬ bE "Purity/3FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPúR¨RJ­ bE "Purity/1FRt4SRqqt0FBt/event_purity"trainer/global_step2:JPõÿÙçJ® bE "Purity/1FRt0SRqqt0FBt/event_purity"trainer/global_step2:JPª/%hN¯ bI &Purity/4FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JPsgïÈ;° b6 jet/accuracy_4_of_4"trainer/global_step2:JPHìðýJ± bE "Purity/3FRt*SRqqt1FBt/event_purity"trainer/global_step2:JP¦€n~;² b6 jet/accuracy_5_of_7"trainer/global_step2:JPÍÛ©çN³ bI &Purity/*FRt1SRqqt0FBt/event_proportion"trainer/global_step2:JP˜Ù³KN´ bI &Purity/*FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JPs¢VëNµ bI &Purity/3FRt4SRqqt1FBt/event_proportion"trainer/global_step2:JPü»ÿUH¶ bC Purity/*FRt3SRqqt*FBt/FBt_purity"trainer/global_step2:JPL3âN· bI &Purity/0FRt3SRqqt2FBt/event_proportion"trainer/global_step2:JPxõJ¸ bE "Purity/2FRt2SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPb:õäN¹ bI &Purity/3FRt4SRqqt4FBt/event_proportion"trainer/global_step2:JP ïaNº bI &Purity/4FRt4SRqqt1FBt/event_proportion"trainer/global_step2:JP+P&H» bC Purity/3FRt*SRqqt*FBt/FBt_purity"trainer/global_step2:JP‰sùH¼ bC Purity/*FRt1SRqqt2FBt/FRt_purity"trainer/global_step2:JPô¼ úN½ bI &Purity/*FRt3SRqqt2FBt/event_proportion"trainer/global_step2:JPðÃ-×N¾ bI &Purity/0FRt1SRqqt0FBt/event_proportion"trainer/global_step2:JPø ·ÑN¿ bI &Purity/1FRt3SRqqt4FBt/event_proportion"trainer/global_step2:JPÔílÑNÀ bI &Purity/2FRt*SRqqt0FBt/event_proportion"trainer/global_step2:JPÐZ´NÁ bI &Purity/3FRt4SRqqt2FBt/event_proportion"trainer/global_step2:JP¤Ó9J bE "Purity/0FRt0SRqqt4FBt/event_purity"trainer/global_step2:JPŒ€ØNHà bC Purity/3FRt2SRqqt1FBt/FBt_purity"trainer/global_step2:JPm—y"NÄ bI &Purity/*FRt1SRqqt1FBt/event_proportion"trainer/global_step2:JP­q rNÅ bI &Purity/*FRt2SRqqt*FBt/event_proportion"trainer/global_step2:JPyØ[HNÆ bI &Purity/0FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JP:mÔ+HÇ bC Purity/1FRt2SRqqt0FBt/FRt_purity"trainer/global_step2:JPÜ4§OJÈ bE "Purity/1FRt3SRqqt0FBt/event_purity"trainer/global_step2:JP´\5NÉ bI &Purity/2FRt0SRqqt1FBt/event_proportion"trainer/global_step2:JP¦½p€JÊ bE "Purity/*FRt3SRqqt1FBt/event_purity"trainer/global_step2:JP g@Ë b; particle/accuracy_7_of_7"trainer/global_step2:JP°f{YJÌ bE "Purity/*FRt3SRqqt0FBt/event_purity"trainer/global_step2:JP•¯ÓPNÍ bI &Purity/1FRt*SRqqt3FBt/event_proportion"trainer/global_step2:JPø¾NÎ bI &Purity/*FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JPî–grJÏ bE "Purity/*FRt1SRqqt2FBt/event_purity"trainer/global_step2:JP¡¾mNÐ bI &Purity/2FRt*SRqqt*FBt/event_proportion"trainer/global_step2:JPDt§tJÑ bE "Purity/2FRt2SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP¢Õ’HÒ bC Purity/2FRt3SRqqt*FBt/FRt_purity"trainer/global_step2:JPÆ´ÅØNÓ bI &Purity/3FRt0SRqqt3FBt/event_proportion"trainer/global_step2:JP“¶ßtNÔ bI &Purity/3FRt3SRqqt3FBt/event_proportion"trainer/global_step2:JPtŸÆÓ;Õ b6 jet/accuracy_4_of_6"trainer/global_step2:JP¦öþãJÖ bE "Purity/1FRt*SRqqt2FBt/SRqqt_purity"trainer/global_step2:JPŸòpN× bI &Purity/4FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JP’¦ÕÂ9Ø b4 particle/accuracy"trainer/global_step2:JP€4ÀJÙ bE "Purity/0FRt2SRqqt2FBt/event_purity"trainer/global_step2:JPþˆHÚ bC Purity/2FRt*SRqqt3FBt/FRt_purity"trainer/global_step2:JPº°JÛ bE "Purity/0FRt1SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP™ÎÄNÜ bI &Purity/2FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JP'—ð7@Ý b; particle/accuracy_2_of_2"trainer/global_step2:JPuy§â@Þ b; particle/accuracy_2_of_3"trainer/global_step2:JPY2Q6@ß b; particle/accuracy_5_of_6"trainer/global_step2:JPBõ)Jà bE "Purity/4FRt0SRqqt*FBt/event_purity"trainer/global_step2:JPñôNá bI &Purity/3FRt*SRqqt3FBt/event_proportion"trainer/global_step2:JPFɺ-Nâ bI &Purity/3FRt0SRqqt*FBt/event_proportion"trainer/global_step2:JPƒ¢Nã bI &Purity/3FRt2SRqqt*FBt/event_proportion"trainer/global_step2:JP9IÙÚJä bE "Purity/3FRt2SRqqt1FBt/event_purity"trainer/global_step2:JPrÊJå bE "Purity/0FRt3SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP˜ü8‚Jæ bE "Purity/3FRt0SRqqt2FBt/event_purity"trainer/global_step2:JPEqµNç bI &Purity/*FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JP¼³ÅJè bE "Purity/3FRt1SRqqt*FBt/event_purity"trainer/global_step2:JP€¤ß±Jé bE "Purity/3FRt1SRqqt1FBt/event_purity"trainer/global_step2:JPmåµÕJê bE "Purity/*FRt4SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP¢ì¹¹Jë bE "Purity/0FRt3SRqqt*FBt/event_purity"trainer/global_step2:JP‘6P.Hì bC Purity/*FRt1SRqqt0FBt/FRt_purity"trainer/global_step2:JP<<ÂJí bE "Purity/0FRt0SRqqt*FBt/event_purity"trainer/global_step2:JPÿšçeHî bC Purity/3FRt2SRqqt*FBt/FRt_purity"trainer/global_step2:JPxU Jï bE "Purity/3FRt*SRqqt2FBt/SRqqt_purity"trainer/global_step2:JPMk µJð bE "Purity/1FRt3SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP³é‹Nñ bI &Purity/2FRt0SRqqt4FBt/event_proportion"trainer/global_step2:JPpÜ‚“Jò bE "Purity/2FRt1SRqqt0FBt/event_purity"trainer/global_step2:JP\ BÖNó bI &Purity/4FRt4SRqqt0FBt/event_proportion"trainer/global_step2:JPBy¥Jô bE "Purity/2FRt4SRqqt0FBt/event_purity"trainer/global_step2:JPLyÜ·Hõ bC Purity/1FRt2SRqqt2FBt/FRt_purity"trainer/global_step2:JPDW¶ÉHö bC Purity/3FRt4SRqqt*FBt/FRt_purity"trainer/global_step2:JPá}¸ H÷ bC Purity/*FRt2SRqqt1FBt/FRt_purity"trainer/global_step2:JP‹^‘Hø bC Purity/1FRt0SRqqt1FBt/FRt_purity"trainer/global_step2:JPµ0n#Hù bC Purity/1FRt0SRqqt3FBt/FBt_purity"trainer/global_step2:JP­" Jú bE "Purity/2FRt*SRqqt1FBt/event_purity"trainer/global_step2:JPQê“dNû bI &Purity/2FRt0SRqqt3FBt/event_proportion"trainer/global_step2:JP‰:"Nü bI &Purity/2FRt2SRqqt4FBt/event_proportion"trainer/global_step2:JP+4uøJý bE "Purity/2FRt4SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPÒ#h¡Jþ bE "Purity/3FRt3SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP¼”œNÿ bI &Purity/0FRt0SRqqt1FBt/event_proportion"trainer/global_step2:JPŒÅ÷&J€ bE "Purity/0FRt0SRqqt2FBt/event_purity"trainer/global_step2:JP^jÏJ bE "Purity/1FRt0SRqqt*FBt/event_purity"trainer/global_step2:JPàCJ‚ bE "Purity/0FRt*SRqqt3FBt/event_purity"trainer/global_step2:JP•.ÓJƒ bE "Purity/*FRt2SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP3@ߺH„ bC Purity/2FRt3SRqqt1FBt/FBt_purity"trainer/global_step2:JPú¦H… bC Purity/3FRt3SRqqt0FBt/FRt_purity"trainer/global_step2:JPAæãH† bC Purity/*FRt2SRqqt3FBt/FBt_purity"trainer/global_step2:JPµ7uJ‡ bE "Purity/2FRt1SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPZÜ0½Nˆ bI &Purity/4FRt*SRqqt1FBt/event_proportion"trainer/global_step2:JP÷ÞíBH‰ bC Purity/1FRt0SRqqt2FBt/FBt_purity"trainer/global_step2:JP²ôA¡NŠ bI &Purity/0FRt*SRqqt*FBt/event_proportion"trainer/global_step2:JP RŠÔJ‹ bE "Purity/*FRt0SRqqt1FBt/event_purity"trainer/global_step2:JPŒ2”¥JŒ bE "Purity/*FRt3SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPáp H bC Purity/*FRt2SRqqt2FBt/FBt_purity"trainer/global_step2:JP«YØHŽ bC Purity/2FRt0SRqqt3FBt/FRt_purity"trainer/global_step2:JPe„!‘J bE "Purity/0FRt1SRqqt0FBt/event_purity"trainer/global_step2:JPÊÆsN bI &Purity/1FRt*SRqqt0FBt/event_proportion"trainer/global_step2:JP=´¾IN‘ bI &Purity/4FRt3SRqqt*FBt/event_proportion"trainer/global_step2:JPº«­&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 :"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.13599546253681183 6 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.126045823097229 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.4000000059604645 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9659090638160706 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 8 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.10092592984437943 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9856727123260498 & Purity/2FRt2SRqqt1FBt/FBt_purity‚1 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.22812499105930328 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.06666667014360428 particle/accuracy_4_of_4‚1 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.11146150529384613 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1042533814907074 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 9 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.053688984364271164 particle/accuracy_1_of_1‚1 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 ("Purity/1FRt1SRqqt0FBt/event_purity‚0 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9952381253242493 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.3194444477558136 jet/accuracy_4_of_7‚0 & Purity/4FRt*SRqqt0FBt/FRt_purity‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.2652766704559326 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9991645812988281 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.16592496633529663 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9673322439193726 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9937499761581421 *"Purity/1FRt*SRqqt3FBt/SRqqt_purity‚0.5 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.20128801465034485 +jet/accuracy_2_of_4‚0.20511487126350403 jet/accuracy_1_of_7‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.9259259700775146 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.16293102502822876 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9965252876281738 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9943494200706482 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9877029657363892 8 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.24433384835720062 + Purity/*FRt2SRqqt2FBt/FRt_purity‚0.1875 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9814515113830566 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9932471513748169 & Purity/1FRt2SRqqt2FBt/FBt_purity‚1 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.04631897434592247 <"Purity/0FRt2SRqqt0FBt/event_purity‚0.0022564581595361233 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 ;"Purity/1FRt*SRqqt*FBt/event_purity‚0.032384809106588364 :"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.14897191524505615 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.11567839235067368 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.1388888955116272 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 +"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.25 particle/accuracy_5_of_7‚1 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.2857142984867096 *"Purity/0FRt2SRqqt3FBt/SRqqt_purity‚0.5 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.3556532859802246 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 :"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.15020574629306793 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 ;"Purity/1FRt0SRqqt0FBt/event_purity‚0.053688984364271164 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 -jet/accuracy_4_of_4‚0.0021547081414610147 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 jet/accuracy_5_of_7‚0 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 :"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.09981103241443634 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 _step‚98 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.1308642029762268 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11169332265853882 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 particle/accuracy_7_of_7‚1 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.021012544631958008 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.2132936418056488 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.08265765011310577 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 jet/accuracy_4_of_6‚0 8"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.300778329372406 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 (particle/accuracy‚0.7182074189186096 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.12495170533657074 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 particle/accuracy_2_of_2‚1 particle/accuracy_2_of_3‚1 particle/accuracy_5_of_6‚1 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 trainer/global_step‚1208 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.24074074625968933 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 % _timestamp‚1.7884458543890753e+09 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.0972222238779068 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.09065309911966324 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8048520684242249 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.09195402264595032 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 ("Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 8 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.20000000298023224 & Purity/3FRt4SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.17855864763259888 8 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.19301100075244904 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 ("Purity/2FRt*SRqqt1FBt/event_purity‚0 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.10000000149011612 -"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.1875 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.29896029829978943 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.06856060773134232 9"Purity/0FRt*SRqqt3FBt/event_purity‚0.1315789520740509 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.23000000417232513 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 & Purity/*FRt2SRqqt3FBt/FBt_purity‚1 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.17082807421684265 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9904144406318665 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6315982341766357 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.14113907516002655 & Purity/*FRt2SRqqt2FBt/FBt_purity‚1 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.12495170533657074 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.3888888955116272 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚0 <"Purity/1FRt1SRqqt*FBt/event_purity‚0.0034840023145079613 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 & Purity/2FRt3SRqqt1FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.18219660222530365 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9960317611694336 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 jet/accuracy_3_of_7‚0 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 8"Purity/0FRt1SRqqt*FBt/event_purity‚0.110958993434906 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.12379663437604904 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.10000000149011612 +jet/accuracy_2_of_6‚0.14743588864803314 particle/accuracy_2_of_7‚1 particle/accuracy_3_of_7‚1 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.07418802380561829 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 )jet/accuracy_2_of_5‚0.270506888628006 6 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.084528848528862 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.3509344458580017 jet/accuracy_4_of_5‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.3194444477558136 jet/accuracy_5_of_6‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.08028130978345871 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.11623268574476242 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.10405953973531723 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.12809523940086365 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.16178162395954132 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 & Purity/4FRt*SRqqt*FBt/FRt_purity‚0 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 :"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.24667392671108246 6 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.989875316619873 *jet/accuracy_1_of_6‚0.8461538553237915 particle/accuracy_3_of_6‚1 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.36366018652915955 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.1240740716457367 :"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.13171768188476562 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_3_of_4‚1 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.1428571492433548 :"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.13376639783382416 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 jet/accuracy_6_of_7‚0 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 :"Purity/0FRt1SRqqt2FBt/event_purity‚0.03910098224878311 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.2208428829908371 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.11681689321994781 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.07407407462596893 ,jet/accuracy_3_of_3‚0.011077145114541054 ,"Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.125 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.2222222238779068 & Purity/1FRt1SRqqt3FBt/FRt_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.14154301583766937 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.18518517911434174 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 ("Purity/1FRt0SRqqt3FBt/event_purity‚0 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9900851249694824 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.125 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.12031156569719315 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.12129595130681992 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9801455736160278 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9785585999488831 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 & Purity/0FRt2SRqqt3FBt/FBt_purity‚1 )loss/val_total_loss‚81.29893085514709 :"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.15789473056793213 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.1965208500623703 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9373522996902466 8 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.20588235557079315 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.15603381395339966 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9981276392936707 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.9298246502876282 particle/accuracy_6_of_6‚1 & Purity/0FRt2SRqqt2FBt/FBt_purity‚1 6 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.982758641242981 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 & Purity/3FRt4SRqqt0FBt/FRt_purity‚0 jet/accuracy_2_of_7‚0 ,"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.125 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9941661357879639 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.2586102783679962 & Purity/2FRt2SRqqt*FBt/FBt_purity‚1 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9833618998527527 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 ;"Purity/0FRt2SRqqt*FBt/event_purity‚0.002910052891820669 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.02511059120297432 8"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.112125925719738 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 8 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.18773402273654938 6 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.953703761100769 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 <"Purity/1FRt1SRqqt2FBt/event_purity‚0.0069444444961845875 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 6validation_average_jet_accuracy‚0.2777352035045624 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.1428571492433548 particle/accuracy_1_of_7‚1 particle/accuracy_4_of_7‚1 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 -"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.1875 7 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.0939393937587738 'particle/f_score‚0.3224147856235504 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_6‚1 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.08510638028383255 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.13047471642494202 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.14249250292778015 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚0 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.12096568197011948 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 *jet/accuracy_1_of_2‚0.5169180035591125 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.1875 :"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.24934537708759308 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 *jet/accuracy_1_of_4‚0.6967126727104187 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.1071428582072258 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.3522167205810547 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.06289467960596085 ("Purity/1FRt*SRqqt3FBt/event_purity‚0 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.1488095223903656 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9922419786453247 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 particle/accuracy_2_of_5‚1 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.1733333170413971 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.05849190428853035 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.2334601879119873 ;"Purity/0FRt2SRqqt1FBt/event_purity‚0.009433962404727936 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 particle/accuracy_2_of_6‚1 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.10612396895885468 ("Purity/2FRt0SRqqt1FBt/event_purity‚0 7 Purity/2FRt0SRqqt1FBt/FBt_purity‚0.9940476417541504 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.18123897910118103 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 ("Purity/2FRt0SRqqt*FBt/event_purity‚0 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.20563571155071259 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 particle/accuracy_1_of_5‚1 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 *"Purity/1FRt1SRqqt3FBt/SRqqt_purity‚0.5 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.14435802400112152 :"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.27548032999038696 :"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.11729497462511063 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.20000000298023224 epoch‚0 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.10448215156793594 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.12167755514383316 particle/accuracy_6_of_7‚1 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9652778506278992 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 ("Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0 :"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.11993966996669769 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.1666666567325592 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 6"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.1165771484375 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.19643500447273254 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 +"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.25 +jet/accuracy_3_of_6‚0.01923076994717121 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.9166666865348816 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9889436364173889 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.1346743255853653 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9968253970146179 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.09884272515773773 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.08873456716537476 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.1466907262802124 ("Purity/2FRt*SRqqt*FBt/event_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 , Purity/1FRt3SRqqt1FBt/FRt_purity‚0.28125 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3074304163455963 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.05490981414914131 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.12021218985319138 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9858024716377258 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9886491298675537 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9981276392936707 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9845058917999268 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 & Purity/1FRt*SRqqt4FBt/FRt_purity‚0 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.09035088121891022 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.18421052396297455 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.20240111649036407 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9404761791229248 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 ."Purity/0FRt0SRqqt3FBt/event_purity‚0.15625 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 8"Purity/*FRt0SRqqt2FBt/event_purity‚0.195530503988266 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.6247178912162781 particle/accuracy_3_of_3‚1 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.06420938670635223 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3124999701976776 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_6‚1 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.04478910565376282 particle/accuracy_1_of_3‚1 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 & Purity/2FRt2SRqqt2FBt/FRt_purity‚0 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.04545454680919647 8"Purity/*FRt0SRqqt3FBt/event_purity‚0.095238097012043 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.10488037019968033 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 8"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.553030252456665 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 *jet/accuracy_1_of_5‚0.7779762148857117 -"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.3125 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 ( Purity/1FRt4SRqqt0FBt/FRt_purity‚0.5 & Purity/4FRt1SRqqt*FBt/FRt_purity‚0 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.09761891514062881 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 jet/accuracy_6_of_6‚0 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.09301326423883438 :"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.23842594027519226 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.12690630555152893 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.11683914065361023 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.09362559020519257 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 & Purity/*FRt*SRqqt4FBt/FRt_purity‚0 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 +jet/accuracy_3_of_5‚0.02800179272890091 & Purity/3FRt0SRqqt2FBt/FRt_purity‚0 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.01720893755555153 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.15416666865348816 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.09550060331821442 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.003968254197388887 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.5227272510528564 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.1944444477558136 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.1666666716337204 jet/accuracy_7_of_7‚0 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 ("Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0 _runtime‚ 969.863100612 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.04496354982256889 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.16692015528678894 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.1242915540933609 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.06883518397808075 particle/accuracy_4_of_5‚1 particle/accuracy_5_of_5‚1 & Purity/*FRt1SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9973544478416443 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.2424854189157486 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9937054514884949 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 )jet/accuracy_1_of_1‚0.290457159280777 *"Purity/*FRt2SRqqt3FBt/SRqqt_purity‚0.5 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.10366613417863846 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.0011773940641433 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6058560013771057 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.1943621039390564 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.2529761791229248 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.16204248368740082 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.1366279125213623 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.05662405863404274 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.04444444552063942 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.13198654353618622 8 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.22499999403953552 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.0972222238779068 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 >&Purity/2FRt*SRqqt2FBt/event_proportioëÏ×nKn‚0.00533234141767025 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 +particle/specificity‚0.7715708613395691 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ,jet/accuracy_3_of_4‚0.021049143746495247 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 8 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.12878789007663727 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 ,"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.125 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.14074072241783142 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.12033744156360626 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.11892447620630264 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.20000000298023224 6 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.998539388179779 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.07503486424684525 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.13267545402050018 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.0833333358168602 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.29553523659706116 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 + Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9375 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9687499403953552 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9833333492279053 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9806202054023743 +jet/accuracy_2_of_3‚0.13349415361881256 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9817273020744324 ("Purity/2FRt*SRqqt0FBt/event_purity‚0 +jet/accuracy_2_of_2‚0.05486276373267174 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 ( Purity/1FRt4SRqqt*FBt/FRt_purity‚0.5 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 & Purity/4FRt1SRqqt0FBt/FRt_purity‚0 ;"Purity/1FRt*SRqqt0FBt/event_purity‚0.022693991661071777 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.2194014936685562 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 7 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.9047619104385376 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.07486670464277267 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt0SRqqt0FBt/event_purity‚0 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.1264367699623108 jet/accuracy_5_of_5‚0 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 particle/accuracy_3_of_5‚1 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.10983243584632874 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0¿ŠÔò§p “£ ("Purity/1FRt3SRqqt0FBt/event_purity‚0 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 particle/accuracy_7_of_7‚1 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.021012544631958008 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.2132936418056488 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.08265765011310577 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 jet/accuracy_4_of_6‚0 8"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.300778329372406 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 (particle/accuracy‚0.7182074189186096 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.12495170533657074 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 particle/accuracy_2_of_2‚1 particle/accuracy_2_of_3‚1 particle/accuracy_5_of_6‚1 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 trainer/global_step‚1208 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.24074074625968933 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 % _timestamp‚1.7884458543890753e+09 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.0972222238779068 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.09065309911966324 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8048520684242249 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.09195402264595032 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 ("Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 8 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.20000000298023224 & Purity/3FRt4SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.17855864763259888 8 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.19301100075244904 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 ("Purity/2FRt*SRqqt1FBt/event_purity‚0 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.10000000149011612 -"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.1875 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.29896029829978943 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.06856060773134232 9"Purity/0FRt*SRqqt3FBt/event_purity‚0.1315789520740509 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.23000000417232513 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 & Purity/*FRt2SRqqt3FBt/FBt_purity‚1 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.17082807421684265 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9904144406318665 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6315982341766357 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.14113907516002655 & Purity/*FRt2SRqqt2FBt/FBt_purity‚1 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.12495170533657074 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.3888888955116272 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚0 <"Purity/1FRt1SRqqt*FBt/event_purity‚0.0034840023145079613 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 & Purity/2FRt3SRqqt1FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.18219660222530365 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9960317611694336 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 jet/accuracy_3_of_7‚0 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 8"Purity/0FRt1SRqqt*FBt/event_purity‚0.110958993434906 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.12379663437604904 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.10000000149011612 +jet/accuracy_2_of_6‚0.14743588864803314 particle/accuracy_2_of_7‚1 particle/accuracy_3_of_7‚1 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.07418802380561829 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 )jet/accuracy_2_of_5‚0.270506888628006 6 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.084528848528862 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.3509344458580017 jet/accuracy_4_of_5‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.3194444477558136 jet/accuracy_5_of_6‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.08028130978345871 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.11623268574476242 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.10405953973531723 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.12809523940086365 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.16178162395954132 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 & Purity/4FRt*SRqqt*FBt/FRt_purity‚0 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 :"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.24667392671108246 6 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.989875316619873 *jet/accuracy_1_of_6‚0.8461538553237915 particle/accuracy_3_of_6‚1 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.36366018652915955 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.1240740716457367 :"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.13171768188476562 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_3_of_4‚1 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.1428571492433548 :"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.13376639783382416 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 jet/accuracy_6_of_7‚0 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 :"Purity/0FRt1SRqqt2FBt/event_purity‚0.03910098224878311 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.2208428829908371 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.11681689321994781 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.07407407462596893 ,jet/accuracy_3_of_3‚0.011077145114541054 ,"Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.125 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.2222222238779068 & Purity/1FRt1SRqqt3FBt/FRt_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.14154301583766937 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.18518517911434174 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 ("Purity/1FRt0SRqqt3FBt/event_purity‚0 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9900851249694824 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.125 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.12031156569719315 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.12129595130681992 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9801455736160278 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9785585999488831 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 & Purity/0FRt2SRqqt3FBt/FBt_purity‚1 )loss/val_total_loss‚81.29893085514709 :"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.15789473056793213 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.1965208500623703 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9373522996902466 8 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.20588235557079315 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.15603381395339966 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9981276392936707 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.9298246502876282 particle/accuracy_6_of_6‚1 & Purity/0FRt2SRqqt2FBt/FBt_purity‚1 6 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.982758641242981 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 & Purity/3FRt4SRqqt0FBt/FRt_purity‚0 jet/accuracy_2_of_7‚0 ,"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.125 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9941661357879639 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.2586102783679962 & Purity/2FRt2SRqqt*FBt/FBt_purity‚1 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9833618998527527 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 ;"Purity/0FRt2SRqqt*FBt/event_purity‚0.002910052891820669 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.02511059120297432 8"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.112125925719738 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 8 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.18773402273654938 6 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.953703761100769 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 <"Purity/1FRt1SRqqt2FBt/event_purity‚0.0069444444961845875 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 6validation_average_jet_accuracy‚0.2777352035045624 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.1428571492433548 particle/accuracy_1_of_7‚1 particle/accuracy_4_of_7‚1 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 -"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.1875 7 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.0939393937587738 'particle/f_score‚0.3224147856235504 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_6‚1 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.08510638028383255 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.13047471642494202 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.14249250292778015 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚0 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.12096568197011948 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 *jet/accuracy_1_of_2‚0.5169180035591125 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.1875 :"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.24934537708759308 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 *jet/accuracy_1_of_4‚0.6967126727104187 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.1071428582072258 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.3522167205810547 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.06289467960596085 ("Purity/1FRt*SRqqt3FBt/event_purity‚0 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.1488095223903656 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9922419786453247 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 particle/accuracy_2_of_5‚1 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.1733333170413971 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.05849190428853035 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.2334601879119873 ;"Purity/0FRt2SRqqt1FBt/event_purity‚0.009433962404727936 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 particle/accuracy_2_of_6‚1 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.10612396895885468 ("Purity/2FRt0SRqqt1FBt/event_purity‚0 7 Purity/2FRt0SRqqt1FBt/FBt_purity‚0.9940476417541504 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.18123897910118103 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 ("Purity/2FRt0SRqqt*FBt/event_purity‚0 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.20563571155071259 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 particle/accuracy_1_of_5‚1 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 *"Purity/1FRt1SRqqt3FBt/SRqqt_purity‚0.5 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.14435802400112152 :"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.27548032999038696 :"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.11729497462511063 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.20000000298023224 epoch‚0 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.10448215156793594 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.12167755514383316 particle/accuracy_6_of_7‚1 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9652778506278992 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 ("Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0 :"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.11993966996669769 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.1666666567325592 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 6"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.1165771484375 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.19643500447273254 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 +"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.25 +jet/accuracy_3_of_6‚0.01923076994717121 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.9166666865348816 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9889436364173889 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.1346743255853653 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9968253970146179 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.09884272515773773 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.08873456716537476 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.1466907262802124 ("Purity/2FRt*SRqqt*FBt/event_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 , Purity/1FRt3SRqqt1FBt/FRt_purity‚0.28125 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3074304163455963 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.05490981414914131 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.12021218985319138 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9858024716377258 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9886491298675537 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9981276392936707 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9845058917999268 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 & Purity/1FRt*SRqqt4FBt/FRt_purity‚0 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.09035088121891022 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.18421052396297455 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.20240111649036407 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9404761791229248 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 ."Purity/0FRt0SRqqt3FBt/event_purity‚0.15625 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 8"Purity/*FRt0SRqqt2FBt/event_purity‚0.195530503988266 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.6247178912162781 particle/accuracy_3_of_3‚1 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.06420938670635223 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3124999701976776 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_6‚1 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.04478910565376282 particle/accuracy_1_of_3‚1 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 & Purity/2FRt2SRqqt2FBt/FRt_purity‚0 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.04545454680919647 8"Purity/*FRt0SRqqt3FBt/event_purity‚0.095238097012043 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.10488037019968033 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 8"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.553030252456665 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 *jet/accuracy_1_of_5‚0.7779762148857117 -"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.3125 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 ( Purity/1FRt4SRqqt0FBt/FRt_purity‚0.5 & Purity/4FRt1SRqqt*FBt/FRt_purity‚0 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.09761891514062881 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 jet/accuracy_6_of_6‚0 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.09301326423883438 :"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.23842594027519226 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.12690630555152893 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.11683914065361023 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.09362559020519257 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 & Purity/*FRt*SRqqt4FBt/FRt_purity‚0 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 +jet/accuracy_3_of_5‚0.02800179272890091 & Purity/3FRt0SRqqt2FBt/FRt_purity‚0 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.01720893755555153 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.15416666865348816 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.09550060331821442 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.003968254197388887 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.5227272510528564 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.1944444477558136 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.1666666716337204 jet/accuracy_7_of_7‚0 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 ("Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0 _runtime‚ 969.863100612 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.04496354982256889 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.16692015528678894 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.1242915540933609 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.06883518397808075 particle/accuracy_4_of_5‚1 particle/accuracy_5_of_5‚1 & Purity/*FRt1SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9973544478416443 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.2424854189157486 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9937054514884949 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 )jet/accuracy_1_of_1‚0.290457159280777 *"Purity/*FRt2SRqqt3FBt/SRqqt_purity‚0.5 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.10366613417863846 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.0011773940641433 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6058560013771057 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.1943621039390564 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.2529761791229248 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.16204248368740082 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.1366279125213623 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.05662405863404274 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.04444444552063942 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.13198654353618622 8 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.22499999403953552 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.0972222238779068 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 +particle/specificity‚0.7715708613395691 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ,jet/accuracy_3_of_4‚0.021049143746495247 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 8 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.12878789007663727 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 ,"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.125 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.14074072241783142 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.12033744156360626 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.11892447620630264 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.20000000298023224 6 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.998539388179779 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.07503486424684525 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.13267545402050018 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.0833333358168602 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.29553523659706116 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 + Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9375 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9687499403953552 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9833333492279053 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9806202054023743 +jet/accuracy_2_of_3‚0.13349415361881256 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9817273020744324 ("Purity/2FRt*SRqqt0FBt/event_purity‚0 +jet/accuracy_2_of_2‚0.05486276373267174 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 ( Purity/1FRt4SRqqt*FBt/FRt_purity‚0.5 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 & Purity/4FRt1SRqqt0FBt/FRt_purity‚0 ;"Purity/1FRt*SRqqt0FBt/event_purity‚0.022693991661071777 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.2194014936685562 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 7 Purity/0FRt°Ekäó 1SRqqt3FBt/FBt_purity‚0.9047619104385376 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.07486670464277267 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt0SRqqt0FBt/event_purity‚0 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.1264367699623108 jet/accuracy_5_of_5‚0 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 particle/accuracy_3_of_5‚1 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.10983243584632874 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.15128472447395325 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.06259359419345856 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 <"Purity/*FRt2SRqqt*FBt/event_purity‚0.0013452477287501097 ;"Purity/1FRt*SRqqt2FBt/event_purity‚0.017579955980181694 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ,particle/sensitivity‚0.42945748567581177 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.3602343201637268 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 , Purity/1FRt0SRqqt3FBt/FRt_purity‚0.21875 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 :"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.28428593277931213 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.09683908522129059 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 particle/accuracy_1_of_4‚1 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.2857142984867096 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.21718750894069672 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.1458333283662796 ("Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 :"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.13599546253681183 6 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.126045823097229 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.4000000059604645 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9659090638160706 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 8 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.10092592984437943 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9856727123260498 & Purity/2FRt2SRqqt1FBt/FBt_purity‚1 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.22812499105930328 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.06666667014360428 particle/accuracy_4_of_4‚1 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.11146150529384613 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1042533814907074 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 9 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.053688984364271164 particle/accuracy_1_of_1‚1 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 ("Purity/1FRt1SRqqt0FBt/event_purity‚0 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9952381253242493 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.3194444477558136 jet/accuracy_4_of_7‚0 & Purity/4FRt*SRqqt0FBt/FRt_purity‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.2652766704559326 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9991645812988281 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.16592496633529663 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9673322439193726 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9937499761581421 *"Purity/1FRt*SRqqt3FBt/SRqqt_purity‚0.5 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.20128801465034485 +jet/accuracy_2_of_4‚0.20511487126350403 jet/accuracy_1_of_7‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.9259259700775146 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.16293102502822876 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9965252876281738 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9943494200706482 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9877029657363892 8 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.24433384835720062 + Purity/*FRt2SRqqt2FBt/FRt_purity‚0.1875 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9814515113830566 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9932471513748169 & Purity/1FRt2SRqqt2FBt/FBt_purity‚1 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.04631897434592247 <"Purity/0FRt2SRqqt0FBt/event_purity‚0.0022564581595361233 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 ;"Purity/1FRt*SRqqt*FBt/event_purity‚0.032384809106588364 :"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.14897191524505615 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.11567839235067368 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.1388888955116272 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 +"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.25 particle/accuracy_5_of_7‚1 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.2857142984867096 *"Purity/0FRt2SRqqt3FBt/SRqqt_purity‚0.5 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.3556532859802246 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 :"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.15020574629306793 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 ;"Purity/1FRt0SRqqt0FBt/event_purity‚0.053688984364271164 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 -jet/accuracy_4_of_4‚0.0021547081414610147 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 jet/accuracy_5_of_7‚0 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 :"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.09981103241443634 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 _step‚98 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.1308642029762268 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11169332265853882b·>žž ‚(:“ £‰æÔ÷½íü gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚146.471' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.294199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€Jg]DŸ ‚(:¹ £‰æÔÅùîÿ$ disk./.usageGB‚297.7290458679199 disk.drbd2080.out‚0 disk.sda3.in‚0% disk.nvme0n1.out‚2890.64794921875 disk.drbd2265.in‚0 disk.sda1.in‚0 disk.drbd2265.out‚0$ memory_percent‚94.14782524108887 cpu‚25.398158981123604 proc.cpu.threads‚17' disk.nvme0c0n1.out‚2890.64794921875 disk.sda.in‚0 disk.sda.out‚ 35.640625 network.recv‚407909 disk.sda3.out‚ 35.640625) proc.memory.percent‚15.93320369720459 network.sent‚ 1.596734e+06 disk.drbd2266.out‚0 disk.rbd1.out‚5.265625 disk.dm-1.out‚ 35.640625 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.sda2.out‚0* disk./.usagePercent‚19.980846079504715 disk.drbd2080.in‚0 disk.drbd2266.in‚0" disk.dm-0.out‚2890.64794921875 disk.dm-1.in‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0' proc.memory.availableMB‚ 958.8203125 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0# proc.memory.rssMB‚ 2610.49609375¶~(Ý“  ‚(:ˆ ²‰æÔ’ö§ý gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.337' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8674 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0!§ÆB¡ ‚(:· ²‰æÔ»éŽÿ% disk./.usageGB‚297.73548889160156 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0$ memory_percent‚94.14815902709961# proc.memory.rssMB‚ 2610.49609375 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.out‚5.265625 disk.drbd2080.out‚0 cpu‚25.39282826310755 disk.sda3.in‚0" disk.dm-0.out‚3823.49560546875 disk.sda.in‚0 disk.sda2.in‚0 disk.dm-1.out‚ 36.234375 disk.dm-0.in‚0.078125 network.recv‚414913' disk.nvme0c0n1.out‚3823.49560546875 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 network.sent‚ 1.720398e+06 disk.rbd0.out‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0& proc.memory.availableMB‚ 958.765625) disk./.usagePercent‚19.98127847623005 disk.sda3.out‚ 36.234375 disk.drbd2266.out‚0 disk.rbd0.in‚0% disk.nvme0n1.out‚3823.49560546875 disk.drbd2080.in‚0 proc.cpu.threads‚17 disk.sda.out‚ 36.234375 disk.drbd1716.in‚0) proc.memory.percent‚15.93320369720459´éú2¢ - _wandbruntime‚1004  _runtime‚1004üË7¿Æ£ À ' lr-AdamW/pg2‚0.00012668146855130246 trainer/global_step‚1249 # _timestamp‚1.78844588910758e+09 _runtime‚1004.579254412 _step‚99 ' lr-AdamW/pg1‚0.00012668146855130246ë¨6¹Ê¤ Ä _runtime‚1004.579254412 _step‚99 ' lr-AdamW/pg1‚0.00012668146855130246 ' lr-AdamW/pg2‚0.00012668146855130246 trainer/global_step‚1249 # _timestamp‚1.78844588910758e+09cOÙƒà2¥ - _wandbruntime‚1004  _runtime‚1004 lÃtÇ'¦ ÁO D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚41816058 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1048070564864 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16701474 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚73826573425152 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6075594 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚1249 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚71913580806911 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157333504 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18208374 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23607684 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚426 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18208376 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚73826573425152 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚72778484857344 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1048070564864 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚80571635166208 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1043555893643 _runtime‚1004.580236836 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6076020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23607684 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18208376 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚71912909329803 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1047414203392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚72778502860288 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚73825899060736 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚73825899060736 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10625454 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚79470248602112 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚428 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚79470224662016 _step‚100 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚41836096 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181273600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656–¶éžË'§ ÅO IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚73825899060736 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10625454 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚79470248602112 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚428 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚79470224662016 _step‚100 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚41836096 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181273600 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚41816058 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1048070564864 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16701474 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚73826573425152 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6075594 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚1249 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚71913580806911 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157333504 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18208374 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚23607684 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚426 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18208376 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚73826573425152 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚72778484857344 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1048070564864 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚80571635166208 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1043555893643 _runtime‚1004.580236836 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6076020 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚23607684 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18208376 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚71912909329803 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1047414203392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚72778502860288 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚73825899060736 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944d"«Ä°|¨ ‚(:ˆ Á‰æÔÎð“ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚126.224' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.2448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncoèc¥rrectedMemoryErrors‚0¨˜(D© ‚(:¹ Á‰æÔÃIJÿ disk.drbd2080.out‚0 disk.sda3.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.rbd1.in‚0) proc.memory.percent‚15.93320369720459 network.sent‚ 1.725567e+06* disk./.usagePercent‚19.981649431147165 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda2.in‚0 cpu‚25.388083165967696 network.recv‚418172% disk.nvme0c0n1.out‚3860.279296875 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 36.7578125 disk.dm-0.out‚3860.279296875 disk.sda.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125# disk.nvme0n1.out‚3860.279296875 disk.dm-1.out‚ 36.7578125$ memory_percent‚94.15109157562256( proc.memory.availableMB‚ 958.28515625% disk./.usageGB‚297.74101638793945# proc.memory.rssMB‚ 2610.49609375 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.rbd1.out‚ 5.4140625 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 36.7578125 disk.sda1.out‚0 disk.dm-1.in‚0z~nI2ª - _wandbruntime‚1005  _runtime‚1005Ž;'¦Ñ&« ËM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚72834837896192 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23646200 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚72834855899136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1048233631232 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10633622 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚79531899339776 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚72834837896192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7884458898129063e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16713703 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚80634332342272 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 trainer/global_step‚1249 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16714141 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6080519 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚73883745905152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18222390 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚72834855899136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41868592 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18222392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚438 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚71969301372643 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚71968629895327 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚1005.285159616 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41848528 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚41848528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚41868592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚70924257566720 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23626138 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1048890006016 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10633620 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161514496 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚73883745905152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1048890006016 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚436 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185454592 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18222392 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1102271488000 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1048233631232 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6080083 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1045026766563 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23626138 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚79531875399680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1102433002496 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10633620 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161514496 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚73883745905152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1048890006016 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚436 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185454592 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18222392 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1102271488000 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1048233631232 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6080083 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1045026766563 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23626138 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚79531875399680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1102433002496 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚72834837896192 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23646200 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚72834855899136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1048233631232 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10633622 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚79531899339776 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚72834837896192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7884458898129063e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16713703 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚80634332342272 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 trainer/global_step‚1249 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16714141 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6080519 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚73883745905152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18222390 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚72834855899136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41868592 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18222392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚438 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚71969301372643 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚71968629895327 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚1005.285159616 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41848528 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚41848528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚41868592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚70924257566720 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23626138 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088e¿S2­ - _wandbruntime‚1005  _runtime‚1005¦BpX® Ò /loss/FRt3/assignment_loss‚3.213063929943328 % _timestamp‚1.7884458898146625e+09 _runtime‚1005.286498022 0loss/FRt1/assignment_loss‚5.2179764697426245 1loss/SRqqt1/assignment_loss‚2.856946093280141 /loss/FBt1/detection_loss‚4.5868203639984335 trainer/global_step‚1249 _step‚102 0loss/FRt4/assignment_loss‚4.3241467183490965 2loss/SRqqt2/assignment_loss‚2.5402335986014335 0loss/FBt1/assignment_loss‚0.9204035965104898 1loss/SRqqt2/detection_loss‚3.3606732189655304 1loss/SRqqt4/detection_loss‚1.9515243170575962 1loss/SRqqt3/assignment_loss‚2.763816809654236 1loss/FBt2/assignment_loss‚0.46848081086146093 0loss/SRqqt1/detection_loss‚2.689461356982952 /loss/FBt2/detection_loss‚6.0195290477652295 /loss/FRt2/assignment_loss‚6.089285121244543 2loss/SRqqt4/assignment_loss‚2.6384431504540973 .loss/FBt4/detection_loss‚3.443413496017456 .loss/FRt3/detection_loss‚2.670024128036296 epoch‚1 0loss/FBt4/assignment_loss‚1.2176133215427398 .loss/FRt1/detection_loss‚3.314607615533628 /loss/FRt2/detection_loss‚3.3858382842120003 1loss/SRqqt3/detection_loss‚1.2620259892778933 /loss/FBt3/detection_loss‚1.8698626922310464 %loss/total_loss‚69.72080508316762 1loss/FBt3/assignment_loss‚0.23528294107522843 .loss/FRt4/detection_loss‚2.681332011830132è“ù\¯ Ö .loss/FRt3/detection_loss‚2.670024128036296 epoch‚1 0loss/FBt4/assignment_loss‚1.2176133215427398 .loss/FRt1/detection_loss‚3.314607615533628 /loss/FRt2/detection_loss‚3.3858382842120003 1loss/SRqqt3/detection_loss‚1.2620259892778933 /loss/FBt3/detection_loss‚1.8698626922310464 %loss/total_loss‚69.72080508316762 1loss/FBt3/assignment_loss‚0.23528294107522843 .loss/FRt4/detection_loss‚2.681332011830132 /loss/FRt3/assignment_loss‚3.213063929943328 % _timestamp‚1.7884458898146625e+09 _runtime‚1005.286498022 0loss/FRt1/assignment_loss‚5.2179764697426245 1loss/SRqqt1/assignment_loss‚2.856946093280141 /loss/FBt1/detection_loss‚4.5868203639984335 trainer/global_step‚1249 _step‚102 0loss/FRt4/assignment_loss‚4.3241467183490965 2loss/SRqqt2/assignment_loss‚2.5402335986014335 0loss/FBt1/assignment_loss‚0.9204035965104898 1loss/SRqqt2/detection_loss‚3.3606732189655304 1loss/SRqqt4/detection_loss‚1.9515243170575962 1loss/SRqqt3/assignment_loss‚2.763816809654236 1loss/FBt2/assignment_loss‚0.46848081086146093 0loss/SRqqt1/detection_loss‚2.689461356982952 /loss/FBt2/detection_loss‚6.0195290477652295 /loss/FRt2/assignment_loss‚6.089285121244543 2loss/SRqqt4/assignment_loss‚2.6384431504540973 .loss/FBt4/detection_loss‚3.443413496017456fè/+I“° ‚(:ˆ ЉæÔ¶šÌþ gpu.0.gpu‚45.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.766' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5532 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0å;J—E± ‚(:º ЉæÔîÆ΀ disk.sda.out‚ 37.37109375 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2266.out‚0 disk.drbd2080.in‚0 disk.dm-1.out‚ 37.37109375( proc.memory.availableMB‚ 958.23046875 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.rbd1.out‚ 5.4140625* disk./.usagePercent‚19.982426158006046# proc.memory.rssMB‚ 2610.49609375) proc.memory.percent‚15.93320369720459 network.sent‚ 1.822849e+06 disk.sda3.out‚ 37.37109375 disk.dm-0.in‚0.078125 disk.dm-1.in‚0 disk.drbd1716.in‚0# memory_percent‚94.1514253616333 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd2266.in‚0 network.recv‚424904 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚3895.494140625 disk.sda3.in‚0 disk.sda.in‚0 disk.rbd0.out‚0 disk.drbd1716.out‚0 cpu‚25.38318897454976% disk./.usageGB‚297.75259017944336 disk.drbd2265.in‚0 disk.dm-0.out‚3895.494140625# disk.nvme0n1.out‚3895.494140625 proc.cpu.threads‚17 disk.rbd0.in‚0Õœ|“² ‚(:ˆ ߉æÔýÜŠý gpu.0.gpu‚22.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚124.182' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.8364 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0yï®/D³ ‚(:¹ ߉æÔÎκÿ disk.rbd1.out‚ 5.5712890625# proc.memory.rssMB‚ 2610.49609375 disk.sda.in‚0$ disk.nvme0c0n1.out‚ 3937.80859375 disk.drbd2266.out‚0 network.sent‚ 1.919228e+06 disk.nvme0c0n1.in‚0.078125$ memory_percent‚94.15366649627686 disk.sda3.out‚ 37.8203125 disk.drbd1716.out‚0 disk.drbd2266.in‚0 network.recv‚429487 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.drbd2265.in‚0 disk.sda3.in‚0 disk.dm-0.out‚ 3937.80859375 disk.rbd1.in‚0* disk./.usagePercent‚19.982747959476587% disk./.usageGB‚297.75738525390625 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.dm-1.out‚ 37.8203125 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda2.out‚0" disk.nvme0n1.out‚ 3937.80859375) proc.memory.percent‚15.93320369720459 cpu‚25.378550018123903 proc.cpu.threads‚17 disk.rbd0.out‚0 disk.sda.out‚ 37.8203125 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0( proc.memory.availableMB‚ 957.86328125 disk.sda1.out‚0 disk.sda2.in‚0~Ó±ì2´ - _wandbruntime‚1042  _runtime‚1042ّǡɵ à % _timestamp‚1.7884459268614209e+09 _runtime‚1042.333119795 _step‚103 ' lr-AdamW/pg1‚0.00012666430868909397 ' lr-AdamW/pg2‚0.00012666430868909397 trainer/global_step‚1299£ÑqͶ Ç % _timestamp‚1.7884459268614209e+09 _runtime‚1042.333119795 _step‚103 ' lr-AdamW/pg1‚0.00012666430868909397 ' lr-AdamW/pg2‚0.00012666430868909397 trainer/global_step‚1299g ¤(¦2· - _wandbruntime‚1042  _runtime‚1042Z³ 8Ǹ ÁO [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚75596136799744 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11033854 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚43442910 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚76684536808960 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚74699622674319 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚74698951197211 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1085045670799 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1088400009216 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚82552761545216 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _runtime‚1042.334133268 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1144454535168 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚76685211173376 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚83697050358272 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚76685211173376 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/reµ{–Õquested_bytes.large_pool.freed‚73614559964160 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1089056370688 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 % _timestamp‚1.7884459268621974e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1089056370688 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚73614577003520 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189662208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165722112 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18909176 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6302438 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24533736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11033852 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚104 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24533736 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17335868 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚43462948 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17336292 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚1299 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚75596136799744 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6302016 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚83697240020480 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚424 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1088400009216 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24553772 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚82552785485312ñ>×zË'¹ ÅO LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1088400009216 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚82552761545216 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _runtime‚1042.334133268 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1144454535168 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚76685211173376 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚83697050358272 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚76685211173376 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚73614559964160 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1089056370688 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 % _timestamp‚1.7884459268621974e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1089056370688 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚73614577003520 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189662208 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165722112 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18909176 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6302438 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24533736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11033852 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚104 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚24533736 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17335868 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚43462948 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚17336292 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚1299 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚75596136799744 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6302016 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚83697240020480 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚424 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1088400009216 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24553772 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚82552785485312 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚75596136799744 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11033854 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚43442910 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚76684536808960 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚74699622674319 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚74698951197211 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1085045670799 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513hX[Ê2º - _wandbruntime‚1043  _runtime‚1043¾#$Ñ&» ËM =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43475380 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24572252 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24552190 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1089219437056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚75652489838592 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚75652489838592 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚82614412282880 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1145158594048 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚75652507841536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1089875811840 trainer/global_step‚1299 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11042022 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚73669464094720 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18923190 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17348513 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚83759754234368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚74755343240051 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17348939 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18923192 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚75652507841536 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚76742383653376 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚82614436222976 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚83759570876928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚1043.300129365 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚74754671762735 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18923190 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43495444 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚73669481134080 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884459278279948e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43495444 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚76742383653376 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24552190 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1089219437056 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11042020 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6306917 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚424 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24572252 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚426 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183357440~¾äƼ ÏM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43495444 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚76742383653376 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24552190 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1089219437056 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11042020 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6306917 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚424 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24572252 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚426 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183357440 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43475380 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24572252 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24552190 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1089219437056 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚75652489838592 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚75652489838592 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚82614412282880 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1145158594048 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚75652507841536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1089875811840 trainer/global_step‚1299 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11042022 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚73669464094720 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18923190 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17348513 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚83759754234368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚74755343240051 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17348939 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18923192 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 MˆÄ-Æ>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚75652507841536 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚76742383653376 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚82614436222976 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚83759570876928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚1043.300129365 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚74754671762735 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18923190 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43495444 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚73669481134080 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884459278279948e+09i£šnÐ2½ - _wandbruntime‚1043  _runtime‚1043µðV¾ Ð 0loss/FRt2/assignment_loss‚5.8782251079877215 2loss/SRqqt3/assignment_loss‚2.6919967348192944 1loss/SRqqt3/detection_loss‚1.2373385802888162 .loss/FBt2/detection_loss‚4.143629366600955 1loss/SRqqt4/assignment_loss‚2.284645772169507 1loss/FBt3/assignment_loss‚0.35183671036497416 -loss/FRt4/detection_loss‚1.95846667728926 trainer/global_step‚1299 /loss/FRt3/assignment_loss‚3.413139858144395 /loss/FRt3/detection_loss‚2.5841029768294477 /loss/FBt1/detection_loss‚3.7760596102213158 /loss/FRt1/detection_loss‚3.3675329147113695 /loss/FRt2/detection_loss‚2.5989829003810883 _runtime‚1043.301290357 2loss/SRqqt2/assignment_loss‚2.7602791587511697 0loss/FBt4/assignment_loss‚1.0938822031021118 .loss/FBt3/detection_loss‚1.538137484010372 _step‚106 0loss/FBt1/assignment_loss‚0.8794303711722878 0loss/SRqqt2/detection_loss‚3.566077121981868 /loss/FBt4/detection_loss‚4.3045507073402405 .loss/FRt1/assignment_loss‚6.45445493194792 /loss/FRt4/assignment_loss‚4.454206032188315 0loss/FBt2/assignment_loss‚0.4623860846920285 % _timestamp‚1.7884459278297546e+09 2loss/SRqqt1/assignment_loss‚2.4557713575141373 1loss/SRqqt1/detection_loss‚2.7550669714461926 1loss/SRqqt4/detection_loss‚1.9654873064940883 %loss/total_loss‚66.97568694044888 epoch‚1»<UZ¿ Ô 2loss/SRqqt2/assignment_loss‚2.7602791587511697 0loss/FBt4/assignment_loss‚1.0938822031021118 .loss/FBt3/detection_loss‚1.538137484010372 _step‚106 0loss/FBt1/assignment_loss‚0.8794303711722878 0loss/SRqqt2/detection_loss‚3.566077121981868 /loss/FBt4/detection_loss‚4.3045507073402405 .loss/FRt1/assignment_loss‚6.45445493194792 /loss/FRt4/assignment_loss‚4.454206032188315 0loss/FBt2/assignment_loss‚0.4623860846920285 % _timestamp‚1.7884459278297546e+09 2loss/SRqqt1/assignment_loss‚2.4557713575141373 1loss/SRqqt1/detection_loss‚2.7550669714461926 1loss/SRqqt4/detection_loss‚1.9654873064940883 %loss/total_loss‚66.97568694044888 epoch‚1 0loss/FRt2/assignment_loss‚5.8782251079877215 2loss/SRqqt3/assignment_loss‚2.6919967348192944 1loss/SRqqt3/detection_loss‚1.2373385802888162 .loss/FBt2/detection_loss‚4.143629366600955 1loss/SRqqt4/assignment_loss‚2.284645772169507 1loss/FBt3/assignment_loss‚0.35183671036497416 -loss/FRt4/detection_loss‚1.95846667728926 trainer/global_step‚1299 /loss/FRt3/assignment_loss‚3.413139858144395 /loss/FRt3/detection_loss‚2.5841029768294477 /loss/FBt1/detection_loss‚3.7760596102213158 /loss/FRt1/detection_loss‚3.3675329147113695 /loss/FRt2/detection_loss‚2.5989829003810883 _runtime‚1043.301290357j2Ä`“À ‚(:ˆ î‰æÔÐȲý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚191.156' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.2312 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‘¨Ÿ@Á ‚(:µ î‰æÔ±¥­ÿ disk.sda3.out‚ 38.40234375 disk.dm-0.out‚ 3960.421875 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0" disk.nvme0c0n1.out‚ 3960.421875 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 network.sent‚ 1.924937e+06 disk.drbd2265.in‚0 cpu‚25.373988222591045 network.recv‚433928 disk.dm-1.in‚0 disk.dm-1.out‚ 38.40234375 disk.drbd1716.out‚0 disk.drbd2265.out‚0$ memory_percent‚94.16234493255615* disk./.usagePercent‚19.979881699123233$ disk./.usageGB‚297.7146759033203 disk.drbd2266.in‚0 disk.sda.out‚ 38.40234375 disk.sda1.out‚0 disk.rbd1.out‚ 5.5712890625 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0 disk.nvme0n1.out‚ 3960.421875 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 956.44140625 disk.sda2.in‚0# proc.memory.rssMB‚ 2610.49609375) proc.memory.percent‚15.93320369720459y Û“ ‚(:ˆ ý‰æÔÏŠÓý gpu.0.gpu‚27.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚142.071' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚28.4142 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0·²á-à ‚(:¢ ý‰æÔß®Âÿ disk.sda.in‚0 disk.sda3.out‚38.75 disk.dm-0.in‚0.078125* disk./.usagePercent‚19.982481455633444 disk.dm-1.out‚38.75 disk.rbd0.out‚0 disk.sda1.out‚0# disk.nvme0n1.out‚4010.927734375 disk.dm-1.in‚0 disk.dm-0.out‚4010.927734375 disk.sda.out‚38.75$ memory_percent‚94.16475296020508 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0 proc.memory.rssMB‚2610.5 disk.drbd2265.in‚0 disk.drbd2266.out‚0& proc.memory.availableMB‚ 956.046875 proc.cpu.threads‚17 network.recv‚439316% disk./.usageGB‚297.75341415405273 disk.nvme0n1.in‚0.078125% disk.nvme0c0n1.out‚4010.927734375 disk.sda2.in‚0 disk.sda2.out‚0 cpu‚25.36953897629794 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd1.out‚ 5.6337890625( proc.memory.percent‚15.9332275390625 network.sent‚ 2.032109e+06 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.rbd0.in‚0y¾ÚCžÄ ‚(:“ ŒŠæÔ–£ý gpu.0.gpu‚50.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚135.934' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.186799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0QmuG?Å ‚(:´ ŒŠæÔʇ¿ÿ disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.dm-0.out‚4045.248046875( proc.memory.percent‚15.9332275390625 network.sent‚ 2.038527e+06 disk.nvme0n1.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0# memory_percent‚94.1669225692749* disk./.usagePercent‚19.982959677707406 disk.dm-1.in‚0% disk.nvme0c0n1.out‚4045.248046875 disk.sda3.out‚ 39.39453125 disk.rbd1.out‚ 5.6337890625 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0# disk.nvme0n1.out‚4045.248046875 disk.dm-1.out‚ 39.39453125 disk.sda3.in‚0( proc.memory.availableMB‚ 955.69140625 disk.drbd1716.out‚0 disk.sda.out‚ 39.39453125 proc.memory.rssMB‚2610.5 cpu‚25.36499871722977 disk.sda.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 proc.cpu.threads‚17 network.recv‚444374$ disk./.usageGB‚297.7605400085449‰ÇÓë2Æ - _wandbruntime‚1080  _runtime‚1080˜Èt:ÉÇ Ã trainer/global_step‚1349 % _timestamp‚1.7884459651818886e+09 _runtime‚1080.653527442 _step‚107 ' lr-AdamW/pg1‚0.00012663395272904818 ' lr-AdamW/pg2‚0.00012663395272904818Õ·ÍÈ Ç ' lr-AdamW/pg1‚0.00012663395272904818 ' lr-AdamW/pg2‚0.00012663395272904818 trainer/global_step‚1349 % _timestamp‚1.7884459651818886e+09 _runtime‚1080.653527442 _step‚107klk^P2É - _wandbruntime‚1080  _runtime‚1080 dÞÇ'Ê ÁO MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚79543848921600 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1129385815040 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11442254 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6529735 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚77484993064619 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1125226572459 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155236352 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚1080.654364442 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11442252 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1187268905472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25479824 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚79543174557184 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19609974 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚78413806745088 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1130042176512 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚85635322368512 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚86822412097536 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚79543174557184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚77485664541727 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚76359783531520 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17971560 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179176448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19609976 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚45069762 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚79543848921600 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚78413788742144 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1130042176512 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19609976 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1125881010207 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚1349 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚78413806745088 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚86822591273984 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45089800 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25479824 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25459788 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1187113669120 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚78413788742144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚76359766492160 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2/Ô’ÏË'Ë ÅO G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1129385815040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚77485664541727 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚76359783531520 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚17971560 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179176448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19609976 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚45069762 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚79543848921600 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚78413788742144 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1130042176512 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19609976 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1125881010207 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚1349 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚78413806745088 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚86822591273984 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45089800 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25479824 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25459788 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1187113669120 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚78413788742144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚76359766492160 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚79543848921600 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1129385815040 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11442254 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6529735 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚77484993064619 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1125226572459 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155236352 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚1080.654364442 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11442252 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1187268905472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25479824 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚79543174557184 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19609974 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚78413806745088 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1130042176512 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚85635322368512 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚86822412097536 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚79543174557184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17984248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1130861617664 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6534271 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚86885111551488 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚79601021401600 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceS]ÂÒ%tatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚109 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚76414670622720 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1130205242880 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17984693 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11450420 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45122296 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚77541385107459 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚76414687662080 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1130205242880 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11450422 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6533828 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1188138445312 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19623992 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19623990 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1126043007423 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45122296 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25498304 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25478242 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _runtime‚1081.359415865 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1126697445379 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19623992 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1130861617664 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25498304 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚77540713630143 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚78470159783936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚85696949166080 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1187974833664 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25478242 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚78470141780992 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚85696973106176 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19623990 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚45102232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884459658871496e+09 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚79600347023872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45102232 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚86884923999744CÅdGÕ&Î ÏM C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45102232 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚86884923999744 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚78470159783936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17984248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1130861617664 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6534271 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚86885111551488 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚79601021401600 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚109 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚76414670622720 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1130205242880 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17984693 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11450420 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45122296 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚77541385107459 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚76414687662080 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1130205242880 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11450422 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6533828 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1188138445312 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19623992 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19623990 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1126043007423 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45122296 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25498304 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25478242 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _runtime‚1081.359415865 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1126697445379 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19623992 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1130861617664 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25498304 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚77540713630143 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚78470159783936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚85696949166080 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1187974833664 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚25478242 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚78470141780992 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚85696973106176 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚19623990 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚45102232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884459658871496e+09 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚79600347023872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430mTŽ ×2Ï - _wandbruntime‚1081  _runtime‚1081Üi¶³WÐ Ñ 0loss/FBt2/assignment_loss‚0.4812525562811061 /loss/FRt2/detection_loss‚2.6500615600881905 -loss/FRt3/detection_loss‚2.67464942600992 /loss/FRt4/detection_loss‚2.9884915094483984 .loss/FBt2/detection_loss‚5.244348183274269 /loss/FBt4/detection_loss‚1.6888518503734045 _step‚110 0loss/FRt2/assignment_loss‚5.2423593504675505 2loss/SRqqt3/assignment_loss‚2.6426479249465755 1loss/FBt3/assignment_loss‚0.28175405815851146 1loss/SRqqt3/detection_loss‚1.5028688936528363 0loss/SRqqt4/detection_loss‚2.380834635746266 2loss/SRqqt4/assignment_loss‚2.3530551120638847 0loss/FBt4/assignment_loss‚0.6211501147065844 _runtime‚1081.360519628 /loss/FRt1/detection_loss‚2.8087026601613956 /loss/FBt1/detection_loss‚3.2879404434197776 %loss/total_loss‚63.29151496353246 0loss/FRt1/assignment_loss‚5.1658707385839415 trainer/global_step‚1349 .loss/FRt4/assignment_loss‚4.32411157543009 0loss/FBt1/assignment_loss‚0.5829281375725422 0loss/FRt3/assignment_loss‚2.9292714993158975 2loss/SRqqt2/assignment_loss‚2.4368597762925286 0loss/SRqqt1/detection_loss‚3.280542287333258 1loss/SRqqt2/detection_loss‚2.8620257241385323 /loss/FBt3/detection_loss‚2.1928878880571574 epoch‚1 $ _timestamp‚1.788445965888945e+09 1loss/SRqqt1/assignment_loss‚2.668049058009838‹ö~Î[Ñ Õ 1loss/SRqqt3/detection_loss‚1.5028688936528363 0loss/SRqqt4/detection_loss‚2.380834635746266 2loss/SRqqt4/assignment_loss‚2.3530551120638847 0loss/FBt4/assignment_loss‚0.6211501147065844 _runtime‚1081.360519628 /loss/FRt1/detection_loss‚2.8087026601613956 /loss/FBt1/detection_loss‚3.2879404434197776 %loss/total_loss‚63.29151496353246 0loss/FRt1/assignment_loss‚5.1658707385839415 trainer/global_step‚1349 .loss/FRt4/assignment_loss‚4.32411157543009 0loss/FBt1/assignment_loss‚0.5829281375725422 0loss/FRt3/assignment_loss‚2.9292714993158975 2loss/SRqqt2/assignment_loss‚2.4368597762925286 0loss/SRqqt1/detection_loss‚3.280542287333258 1loss/SRqqt2/detection_loss‚2.8620257241385323 /loss/FBt3/detection_loss‚2.1928878880571574 epoch‚1 $ _timestamp‚1.788445965888945e+09 1loss/SRqqt1/assignment_loss‚2.668049058009838 0loss/FBt2/assignment_loss‚0.4812525562811061 /loss/FRt2/detection_loss‚2.6500615600881905 -loss/FRt3/detection_loss‚2.67464942600992 /loss/FRt4/detection_loss‚2.9884915094483984 .loss/FBt2/detection_loss‚5.244348183274269 /loss/FBt4/detection_loss‚1.6888518503734045 _step‚110 0loss/FRt2/assignment_loss‚5.2423593504675505 2loss/SRqqt3/assignment_loss‚2.6426479249465755 1loss/FBt3/assignment_loss‚0.28175405815851146n¶€‘›“Ò ‚(:ˆ ›ŠæÔ¤§¤ý gpu.0.gpu‚23.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚162.442' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚32.4884 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚01…P&FÓ ‚(:» ›ŠæÔ­Ãÿ disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0( proc.memory.percent‚15.9332275390625 network.sent‚ 2.043602e+06 disk.rbd0.in‚0 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0% disk.nvme0n1.out‚4086.77197265625 proc.memory.rssMB‚2610.5 network.recv‚447560% disk./.usageGB‚297.76480865478516" disk.dm-0.out‚4086.77197265625 disk.sda.in‚0) disk./.usagePercent‚19.98324615013822 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚4086.77197265625 disk.rbd1.in‚0 disk.dm-1.out‚ 39.78515625' proc.memory.availableMB‚ 955.1953125 cpu‚25.360789243935102 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.sda2.out‚0 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 39.78515625 disk.drbd2266.out‚0 disk.rbd1.out‚ 5.8212890625$ memory_percent‚94.16995048522949 disk.dm-0.in‚0.078125 disk.sda.out‚ 39.78515625 disk.sda1.in‚0 disk.sda1.out‚0ÉùÅ“Ô ‚(:ˆ ªŠæÔ¼Æúü gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.627' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9254 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0N¯>5Õ ‚(:ª ªŠæÔ䨵€" disk.dm-0.out‚4120.56103515625 disk.drbd2265.out‚0 disk.sda3.out‚40.375 disk.sda2.out‚0 network.sent‚ 2.152105e+06( disk./.usagePercent‚19.9837110598204 disk.drbd2266.in‚0 disk.rbd1.out‚ 5.8212890625% disk.nvme0n1.out‚4120.56103515625 disk.drbd1716.in‚0 cpu‚25.35665674650341 disk.drbd2080.in‚0$ memory_percent‚94.17052268981934 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda.out‚40.375 disk.sda1.in‚0 disk.sda1.out‚0' proc.memory.availableMB‚ 955.1015625 disk.rbd1.in‚0 network.recv‚454494% disk./.usageGB‚297.77173614501953 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.drbd2265.in‚0 proc.memory.rssMB‚2610.5 disk.nvme0c0n1.in‚0.078125( proc.memory.percent‚15.9332275390625 disk.sda3.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd1716.out‚0 proc.cpu.threads‚17 disk.dm-1.out‚40.375 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚4120.56103515625 disk.rbd0.in‚0òq 2Ö - _wandbruntime‚1119  _runtime‚1119¯Þ¬É× Ã % _timestamp‚1.7884460036499717e+09 _runtime‚1119.121566259 _step‚111 ' lr-AdamW/pg1‚0.00012659040699730174 ' lr-AdamW/pg2‚0.00012659040699730174 trainer/global_step‚1399ýÁøÍØ Ç ' lr-AdamW/pg2‚0.00012659040699730174 trainer/global_step‚1399 % _timestamp‚1.7884460036499717e+09 _runtime‚1119.121566259 _step‚111 ' lr-AdamW/pg1‚0.00012659040699730174oÝ42Ù - _wandbruntime‚1119  _runtime‚1119Éè%"Ú ÀO SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1166061911867 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11850654 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚82401812305408 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚88717859251712 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1229951413248 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚112 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1170371620864 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚82402486669824 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26405876 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚81231458687488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20310774 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚89947974289920 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26385840 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚80271034932027 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20310776 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20310776 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKD"½S‡¡eviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚79104990059520 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚82401812305408 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1170371620864 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18607263 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18606816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚89947786724864 trainer/global_step‚1399|ðˆÙÊ'Û ÄO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚79104990059520 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚82401812305408 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1170371620864 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18607263 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18606816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚89947786724864 trainer/global_step‚1399 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1166061911867 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11850654 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚82401812305408 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚88717859251712 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1229951413248 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚112 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1170371620864 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚82402486669824 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26405876 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚81231458687488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20310774 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚89947974289920 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26385840 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚80271034932027 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20310776 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20310776 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944p‹D*ë2Ü - _wandbruntime‚1119  _runtime‚1119^èBÑ&Ý ËM FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18619451 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚88779509989376 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1171191048704 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1171847423488 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18619896 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚90010479269376 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚1119.841677437 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1171847423488 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚79159877150720 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20324792 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚1399 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚46729084 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195940352 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172000256 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26404294 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _step‚113 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20324792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6760631 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1167532784787 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26424356 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1230969280000 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚79159894190080 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚81287811726336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884460043693857e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚88779486049280 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11858820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚81287793723392 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚82458984772096 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46749148 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20324790 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024o‘…Õ&Þ ÏM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1171191048704 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11858822 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚46749148 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚80327426974867 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚82459659149824 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20324790 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6761074 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1166878346831 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚1399 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚46729084 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195940352 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172000256 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26404294 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _step‚113 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20324792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6760631 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1167532784787 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26424356 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1230969280000 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚79159894190080 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚81287811726336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884460043693857e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚88779486049280 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11858820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚81287793723392 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚82458984772096 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46749148 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20324790 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18619451 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚88779509989376 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1171191048704 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1171847423488 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18619896 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚90010479269376 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚1119.841677437 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1171847423488 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚79159877150720 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20324792 E´=Éé à ' lr-AdamW/pg1‚0.00012653368056871983 ' lr-AdamW/pg2‚0.00012653368056871983 trainer/global_step‚1449 % _timestamp‚1.7884460421680064e+09 _runtime‚1157.639572769 _step‚115Ó”¼Íê Ç ' lr-AdamW/pg1‚0.00012653368056871983 ' lr-AdamW/pg2‚0.00012653368056871983 trainer/global_step‚1449 % _timestamp‚1.7884460421680064e+09 _runtime‚1157.639572769 _step‚115si@$ 2ë - _wandbruntime‚1157  _runtime‚1157ZPR•Ç'ì ÁO LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚84049110629888 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚85261124418048 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚91800396134912 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6984997 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚81850196587520 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1206897251275 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27311892 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚83057076799435 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6985431 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚93073484560384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1211357426688 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚48323466 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27331928 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21011576 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21011574 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19244049 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚93073290703872 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169916416 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12259052 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21011574 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚48343504 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193856512 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚84049092626944 _runtime‚1157.640506501 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚85261124418048 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27331928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27311892 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1211357426688 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19244485 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1273088425472 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚83057748276543 trainer/global_step‚1449 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚116 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚84049092626944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚436 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12259054 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚434 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚85260450053632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1212013788160 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚81850179548160 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚85260450053632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚84049110629888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21011576 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚91800372194816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1211357426688 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19244485 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1273088425472 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚83057748276543 trainer/global_step‚1449 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚116 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚84049092626944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚436 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12259054 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚434 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚85260450053632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1212013788160 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚81850179548160 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚85260450053632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚84049110629888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21011576 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚91800372194816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚84049110629888 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚85261124418048 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚91800396134912 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6984997 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚81850196587520 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1206897251275 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚27311892 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚83057076799435 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6985431 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚93073484560384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944tE´¯82î - _wandbruntime‚1158  _runtime‚1158 õD/äï ÊM ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84105463668736 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21025592 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12267222 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191746048 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1273939154944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚442 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1208368124195 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1212833229312 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚93135794281472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167805952 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21025590 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1207713686239 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1273771348992 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84105445665792 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1212176854528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19257199 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19256757 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚93135986027520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚84105463668736 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1212833229312 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1158.379776751 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21025592 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚91862022932480 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚84105445665792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚81905083678720 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚85318296898048 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27330346 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚85318296898048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27330346 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48376000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21025590 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48355936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27350408 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088+$Û]Ô&ð ÎM MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚93135986027520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚84105463668736 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1212833229312 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚1158.379776751 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21025592 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚91862022932480 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚84105445665792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚81905083678720 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚85318296898048 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27330346 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚85318296898048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27330346 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48376000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21025590 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48355936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27350408 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84105463668736 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21025592 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12267222 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191746048 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1273939154944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚442 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1208368124195 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1212833229312 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚93135794281472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167805952 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21025590 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1207713686239 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1273771348992 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84105445665792 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1212176854528 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19257199 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19256757 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0uÝÄ2ñ - _wandbruntime‚1158  _runtime‚1158" ¬Tò Î epoch‚1 0loss/FRt3/assignment_loss‚3.4057509354923083 2loss/SRqqt2/assignment_loss‚2.1426213033497334 0loss/SRqqt2/detection_loss‚2.637031339108944 /loss/FBt1/detection_loss‚2.1918844887036664 2loss/SRqqt3/assignment_loss‚2.7918508779257536 0loss/FBt1/assignment_loss‚0.8505812872733388 -loss/FRt3/detection_loss‚2.62598626833895 0loss/SRqqt1/detection_loss‚2.557139829891484 .loss/FRt4/detection_loss‚2.368174421633112 1loss/SRqqt3/detection_loss‚2.1083492542384192 /loss/FBt4/detection_loss‚1.3488944053649903 trainer/global_step‚1449 _runtime‚1158.381019917 1loss/SRqqt1/assignment_loss‚2.536130733606292 2loss/SRqqt4/assignment_loss‚2.0256481571839404 0loss/FBt4/assignment_loss‚1.2706578380428255 &loss/total_loss‚61.775065787788066 /loss/FBt3/detection_loss‚1.7948540560718553 .loss/FRt1/assignment_loss‚4.31711804035098 /loss/FRt2/assignment_loss‚5.473266132946672 1loss/FBt3/assignment_loss‚0.25952322318839494 0loss/SRqqt4/detection_loss‚2.296386499101153 _step‚118 % _timestamp‚1.7884460429094524e+09 .loss/FRt2/detection_loss‚2.854328482315458 /loss/FRt4/assignment_loss‚4.877157342844996 0loss/FBt2/assignment_loss‚0.6983812096998463 /loss/FRt1/detection_loss‚2.6972492163957553 .loss/FBt2/detection_loss‚5.646100444719195©kê"Xó Ò epoch‚1 0loss/FRt3/assignment_loss‚3.4057509354923083 2loss/SRqqt2/assignment_loss‚2.1426213033497334 0loss/SRqqt2/detection_loss‚2.637031339108944 /loss/FBt1/detection_loss‚2.1918844887036664 2loss/SRqqt3/assignment_loss‚2.7918508779257536 0loss/FBt1/assignment_loss‚0.8505812872733388 -loss/FRt3/detection_loss‚2.62598626833895 0loss/SRqqt1/detection_loss‚2.557139829891484 .loss/FRt4/detection_loss‚2.368174421633112 1loss/SRqqt3/detection_loss‚2.1083492542384192 /loss/FBt4/detection_loss‚1.3488944053649903 trainer/global_step‚1449 _runtime‚1158.381019917 1loss/SRqqt1/assignment_loss‚2.536130733606292 2loss/SRqqt4/assignment_loss‚2.0256481571839404 0loss/FBt4/assignment_loss‚1.2706578380428255 &loss/total_loss‚61.775065787788066 /loss/FBt3/detection_loss‚1.7948540560718553 .loss/FRt1/assignment_loss‚4.31711804035098 /loss/FRt2/assignment_loss‚5.473266132946672 1loss/FBt3/assignment_loss‚0.25952322318839494 0loss/SRqqt4/detection_loss‚2.296386499101153 _step‚118 % _timestamp‚1.7884460429094524e+09 .loss/FRt2/detection_loss‚2.854328482315458 /loss/FRt4/assignment_loss‚4.877157342844996 0loss/FBt2/assignment_loss‚0.6983812096998463 /loss/FRt1/detection_loss‚2.6972492163957553 .loss/FBt2/detection_loss‚5.646100444719195vºU¿Ê“ô ‚(:ˆ æŠæԯ܎ý gpu.0.gpu‚35.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.724' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.9448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0À_cžAõ ‚(:¶ æŠæÔüŒœ€ proc.cpu.threads‚17 network.sent‚ 2.277434e+06 disk.rbd0.out‚0 disk.dm-1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0) proc.memory.percent‚15.93325138092041 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2080.in‚0# proc.memory.rssMB‚ 2610.50390625% disk./.usageGB‚297.80063247680664 disk.sda2.in‚0 disk.drbd2265.out‚0 disk.sda3.out‚ 42.35546875 disk.dm-1.out‚ 42.35546875 disk.nvme0c0n1.in‚0.078125 cpu‚25.341322172254582$ disk.nvme0c0n1.out‚ 4288.82421875) disk./.usagePercent‚19.98565031689227 disk.dm-0.out‚ 4288.82421875 disk.sda.out‚ 42.35546875" disk.nvme0n1.out‚ 4288.82421875% proc.memory.availableMB‚ 237.65625 network.recv‚473498 disk.sda.in‚0 disk.drbd1716.out‚0 disk.sda3.in‚0# memory_percent‚98.5494613647461 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.out‚ 6.041015625 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd1.in‚0÷^¨“ö ‚(:ˆ õŠæÔÓÞúü gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.123' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8246 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0QÓÏ•M÷ ‚(: õŠæÔ®ßÙÿ) proc.memory.percent‚15.93325138092041 network.sent‚ 2.384507e+06 disk.drbd2266.in‚0' disk.nvme0c0n1.out‚4327.68115234375 disk.sda3.out‚ 42.80859375 disk.sda.in‚0% disk.nvme0n1.out‚4327.68115234375 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125( proc.memory.availableMB‚ 237.63671875" disk.dm-0.out‚4327.68115234375$ disk./.usageGB‚297.8045349121094 disk.rbd0.out‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 proc.cpu.threads‚17 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.drbd2080.out‚0$ memory_percent‚98.54958057403564 disk.dm-1.in‚0 disk.dm-1.out‚ 42.80859375 disk.drbd1716.in‚0 disk.dm-0.in‚0.078125 network.recv‚478358( disk./.usagePercent‚19.9859122125998 disk.sda3.in‚0 disk.sda2.out‚0 disk.drbd2265.in‚0 disk.rbd1.out‚ 6.0732421875# proc.memory.rssMB‚ 2610.50390625 disk.sda.out‚ 42.80859375 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 cpu‚25.337539792361934 disk.sda1.in‚0 disk.drbd2265.out‚0q¢ÜP2ø - _wandbruntime‚1196  _runtime‚1196Ô9&Éù à ' lr-AdamW/pg1‚0.00012646378526500512 ' lr-AdamW/pg2‚0.00012646378526500512 trainer/global_step‚1499 % _timestamp‚1.7884460814621713e+09 _runtime‚1196.933781959 _step‚119]ý÷ºÍú Ç _step‚119 ' lr-AdamW/pg1‚0.00012646378526500512 ' lr-AdamW/pg2‚0.00012646378526500512 trainer/global_step‚1499 % _timestamp‚1.7884460814621713e+09 _runtime‚1196.933781959wÕE2û - _wandbruntime‚1196  _runtime‚1196bÖ6Æ'ü ÀO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21712376 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1248387028431 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49970356 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚88119087801856 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7214228 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12667454 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚94882909078016 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚94882933018112 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚84595386076160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1252343232512 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1316115444736 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚88119762166272 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189662208 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7213775 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19881682 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚85843118666843 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚96199048462848 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1252343232512 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1252999593984 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚88119762166272 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚88119087801856 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12667452 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚85843790143951 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1247732590683 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 $ _timestamp‚1.788446081462948e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28237944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21712374 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚86866744569344 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚1196.934798508 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19881227 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21712376 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚86866762572288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1315949722624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚84595403115520 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49950318 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944‹(“¸…ý ÄO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1252999593984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28257980 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21712374 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚1499 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1316115444736 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚88119762166272 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189662208 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚120 TGDeviceStatsMonitor.on_train_batch_start!–¯mE /active_bytes.large_pool.current‚18002944 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7213775 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19881682 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚85843118666843 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚96199048462848 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1252343232512 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1252999593984 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚88119762166272 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚88119087801856 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12667452 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚85843790143951 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1247732590683 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 $ _timestamp‚1.788446081462948e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28237944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21712374 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚86866744569344 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚1196.934798508 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19881227 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚21712376 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚86866762572288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1315949722624 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚84595403115520 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49950318 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21712376 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1248387028431 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49970356 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚88119087801856 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7214228 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12667454 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚94882909078016 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚94882933018112 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚84595386076160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1252343232512 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _step‚121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21726392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1316977635840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28276460 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚94944559815680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1249203463603 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1253162660352 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚86923097608192 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚96261561391616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21726392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚86923097608192 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1316820315648 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50002852 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21726390 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚96261380131328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚86923115611136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1253819035136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚94944583755776 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚84650290206720 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚49982788 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1253162660352 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚84650307246080 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚86923115611136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7218868 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚85899510709683 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28256398 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1253819035136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19894490 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7218431 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1248549025647 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚1197.661744643 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚88176934646272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19894051 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12675620 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181260288 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21726390 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12675622 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28256398 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28276460NDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1253819035136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19894490 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7218431 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1248549025647 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚1197.661744643 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚88176934646272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19894051 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12675620 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181260288 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21726390 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12675622 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28256398 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28276460 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 $ _timestamp‚1.788446082189681e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚49982788 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚50002852 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _step‚121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21726392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1316977635840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28276460 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚94944559815680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1249203463603 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1253162660352 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚86923097608192 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚96261561391616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21726392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚86923097608192 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1316820315648 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50002852 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21726390 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚96261380131328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚86923115611136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1253819035136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚94944583755776 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚84650290206720 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚49982788 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1253162660352 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚84650307246080 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚86923115611136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7218868 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚85899510709683y9Ç W2- _wandbruntime‚1197  _runtime‚1197-DñS‚Í /loss/FRt1/assignment_loss‚5.203345303839826 0loss/FBt4/assignment_loss‚0.5502827380384717 /loss/FRt3/assignment_loss‚2.849957040711945 0loss/FBt3/assignment_loss‚0.3303067583777767 .loss/FRt2/detection_loss‚2.862633938139135 1loss/SRqqt3/detection_loss‚1.6433925712349968 epoch‚1 trainer/global_step‚1499 1loss/SRqqt1/assignment_loss‚2.902187132835388 .loss/FRt4/detection_loss‚2.271508795342275 /loss/SRqqt1/detection_loss‚2.89824294646581 /loss/FBt2/detection_loss‚3.7514947590728602 &loss/total_loss‚59.550528272115876 1loss/SRqqt3/assignment_loss‚2.398256804321853 0loss/FBt1/assignment_loss‚0.5764376644272033 /loss/FBt3/detection_loss‚1.3317659615793012 _step‚122 -loss/FRt1/detection_loss‚2.50511808534886 .loss/FRt3/detection_loss‚2.168080814328848 /loss/FBt1/detection_loss‚3.5633786413820063 _runtime‚1197.663218535 2loss/SRqqt4/assignment_loss‚2.2941360971699023 1loss/FBt2/assignment_loss‚0.48982850599395533 1loss/SRqqt2/detection_loss‚2.5605392412441534 0loss/SRqqt2/assignment_loss‚2.71795312733185 1loss/SRqqt4/detection_loss‚1.8156685361715212 /loss/FRt2/assignment_loss‚5.483963446183638 0loss/FRt4/assignment_loss‚4.4097485946757455 /loss/FBt4/detection_loss‚1.9723007678985598 % _timestamp‚1.7884460821917105e+09º¶‹àWƒÑ &loss/total_loss‚59.550528272115876 1loss/SRqqt3/assignment_loss‚2.398256804321853 0loss/FBt1/assignment_loss‚0.5764376644272033 /loss/FBt3/detection_loss‚1.3317659615793012 _step‚122 -loss/FRt1/detection_loss‚2.50511808534886 .loss/FRt3/detection_loss‚2.168080814328848 /loss/FBt1/detection_loss‚3.5633786413820063 _runtime‚1197.663218535 2loss/SRqqt4/assignment_loss‚2.2941360971699023 1loss/FBt2/assignment_loss‚0.48982850599395533 1loss/SRqqt2/detection_loss‚2.5605392412441534 0loss/SRqqt2/assignment_loss‚2.71795312733185 1loss/SRqqt4/detection_loss‚1.8156685361715212 /loss/FRt2/assignment_loss‚5.483963446183638 0loss/FRt4/assignment_loss‚4.4097485946757455 /loss/FBt4/detection_loss‚1.9723007678985598 % _timestamp‚1.7884460821917105e+09 /loss/FRt1/assignment_loss‚5.203345303839826 0loss/FBt4/assignment_loss‚0.5502827380384717 /loss/FRt3/assignment_loss‚2.849957040711945 0loss/FBt3/assignment_loss‚0.3303067583777767 .loss/FRt2/detection_loss‚2.862633938139135 1loss/SRqqt3/detection_loss‚1.6433925712349968 epoch‚1 trainer/global_step‚1499 1loss/SRqqt1/assignment_loss‚2.902187132835388 .loss/FRt4/detection_loss‚2.271508795342275 /loss/SRqqt1/detection_loss‚2.89824294646581 /loss/FBt2/detection_loss‚3.7514947590728602zpÃxž„‚(:“ „‹æÔÁÊžý gpu.0.gpu‚49.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚146.569' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.313799999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÖÉ-6M…‚(: „‹æԣĻÿ disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚4361.47021484375 disk.drbd2265.in‚0 disk.sda3.out‚ 43.44921875 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.dm-1.out‚ 43.44921875( proc.memory.availableMB‚ 237.54296875 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda1.in‚0$ memory_percent‚98.55015277862549 proc.cpu.threads‚17 cpu‚25.33404107720255$ disk./.usageGB‚297.8100776672363 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚4361.47021484375" disk.dm-0.out‚4361.47021484375 disk.sda2.in‚0) proc.memory.percent‚15.93325138092041) disk./.usagePercent‚19.98628419154705 disk.drbd1716.out‚0 disk.rbd1.out‚ 6.0732421875 disk.dm-1.in‚0# proc.memory.rssMB‚ 2610.50390625 network.recv‚483020 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.sda1.out‚0 network.sent‚ 2.390989e+06 disk.sda3.in‚0 disk.sda.out‚ 43.44921875 disk.rbd1.in‚0àؔ؆‚(:ˆ “‹æÔè°™ý gpu.0.gpu‚45.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.482' gpu.0.Ö:IE»enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8964 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿Ç9M‡‚(: “‹æÔåû¤ÿ disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚4397.22802734375 disk.drbd2266.in‚0 disk.dm-1.in‚0) disk./.usagePercent‚19.98682462345183 disk.sda3.in‚0 disk.rbd1.out‚ 6.3701171875 disk.drbd1716.out‚0% disk./.usageGB‚297.81813049316406 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0" disk.dm-0.out‚4397.22802734375 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd2265.out‚0$ memory_percent‚98.55151176452637 network.sent‚ 2.497999e+06 disk.drbd2080.out‚0 disk.rbd1.in‚0) proc.memory.percent‚15.93325138092041 proc.cpu.threads‚17 disk.sda3.out‚ 43.87109375 disk.sda1.out‚0% disk.nvme0n1.out‚4397.22802734375' proc.memory.availableMB‚ 237.3203125# proc.memory.rssMB‚ 2610.50390625 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.sda.in‚0 disk.dm-1.out‚ 43.87109375 disk.drbd2265.in‚0 cpu‚25.33062397370561 network.recv‚488171 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 43.87109375 disk.sda2.in‚0⇓ˆ‚(:ˆ ¢‹æÔ²îüü gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.509' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.7018 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Û ™J‰‚(:¿ ¢‹æÔëÂ…ÿ disk.drbd1716.out‚0% proc.memory.availableMB‚ 237.28125 network.sent‚ 2.504721e+06( disk./.usagePercent‚19.9874464657525 disk.sda1.out‚0 disk.drbd2266.in‚0% disk./.usageGB‚297.82739639282227" disk.dm-0.out‚4430.25146484375 disk.dm-1.in‚0 disk.dm-1.out‚ 44.61328125 disk.rbd0.in‚0 proc.cpu.threads‚17 network.recv‚493412 disk.rbd1.in‚0% disk.nvme0n1.out‚4430.25146484375 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0" proc.memory.rssMB‚ 2610.5078125 disk.sda.out‚ 44.61328125 disk.drbd2080.out‚0 disk.sda3.out‚ 44.61328125$ memory_percent‚98.55175018310547) proc.memory.percent‚15.93327522277832 disk.sda1.in‚0 disk.sda3.in‚0 disk.rbd0.out‚0' disk.nvme0c0n1.out‚4430.25146484375 disk.sda.in‚0 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 cpu‚25.327291477349537 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 6.5185546875k9U˜2Š- _wandbruntime‚1235  _runtime‚1235’ÔQÈ‹Â trainer/global_step‚1549 % _timestamp‚1.7884461200440662e+09 _runtime‚ 1235.51563958 _step‚123 ' lr-AdamW/pg1‚0.00012638073565223422 ' lr-AdamW/pg2‚0.00012638073565223422ˆÙHýÌŒÆ % _timestamp‚1.7884461200440662e+09 _runtime‚ 1235.51563958 _step‚123 ' lr-AdamW/pg1‚0.00012638073565223422 ' lr-AdamW/pg2‚0.00012638073565223422 trainer/global_step‚1549{ø?‚2- _wandbruntime‚1235  _runtime‚1235 ‰ƒÉÇ'ŽÁO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1293985399808 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 _step‚124 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚89684414514688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13075854 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155236352 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚88629160534251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29163996 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1293329038336 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1289222367839 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚90978399914496 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚99324547243008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚89684414514688 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20518551 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1359077341696 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚99324368066560 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚51597208 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1358922105344 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 trainer/global_step‚1549 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22413174 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚420 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13075852 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29163996 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚90977725550080 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚97965445961216 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚89684396511744 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22413176 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚88629832011359Õç³9Ë'ÅO TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚90977725550080 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20518973 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚90978399914496 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚97965469901312 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚89684396511744 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚87340609643520 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚87340592604160 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚51577170 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚22413174 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚420 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13075852 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29163996 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚90977725550080 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚97965445961216 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚89684396511744 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22413176 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚88629832011359 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1293985399808 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 _step‚124 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚89684414514688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13075854 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155236352 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚88629160534251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29163996 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1293329038336 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1289222367839 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚90978399914496 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚99324547243008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚89684414514688 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20518551 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1359077341696 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚99324368066560 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚51597208 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1358922105344 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 trainer/global_step‚1549 IDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚91035572394496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚91035572394496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88684881099775 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚87395513774080 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29182450 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29202512 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚99386859384832 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98027096698880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1359940977664 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1359762685952 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1294804840960 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13084020 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884461207561061e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚51609640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 trainer/global_step‚1549 _step‚125 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚87395496734720 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚89740767553536 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚91034898016768 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1294804840960 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20531226 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1294148466176 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚99387061616640 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1294148466176 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7447647 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 202231808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚89740749550592 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚88685552577091 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51629704 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚441 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51609640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29182450 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚98027120638976 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22427190 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1289384365055 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22427190 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51629704 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1290038803011 OBDeviceStatsMonitor.on_train_batch_end/inavÌUîÆctive_split.all.allocated‚20531669 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚443 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 178291712 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚91035572394496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚91035572394496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88684881099775 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚87395513774080 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29182450 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29202512 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚99386859384832 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98027096698880 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1359940977664 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1359762685952 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1294804840960 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13084020 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884461207561061e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚51609640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 trainer/global_step‚1549 _step‚125 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚87395496734720 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚89740767553536 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚91034898016768 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1294804840960 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20531226 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1294148466176 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚99387061616640 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1294148466176 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7447647 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 202231808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚89740749550592 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚88685552577091 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51629704 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚441 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51609640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29182450 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚98027120638976 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22427190 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1289384365055 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22427190 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51629704 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1290038803011 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20531669 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚443 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 178291712 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚90085799132160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚102450009562624 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚92502048454144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚90085816171520 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23113976 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21155701 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 $ _timestamp‚1.788446158384952e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚53224060 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚93836363298304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7671900 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1401841250304 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚91415873878767 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1334314844160 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚92502066457088 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23113974 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1334971205632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚93836363298304 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21156154 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430s§•h®ŸÈO XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚93837037662720 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1334314844160 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30110084 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1330057707247 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13484254 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185467904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 trainer/global_step‚1599 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚92502066457088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1329403269499 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1334971205632 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7671449 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚101048006784512 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚101047982844416 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚92502048454144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161527808 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚102449824094720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚128 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚92502066457088 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23113974 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1334971205632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allgeªl ocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚93836363298304 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21156154 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30090048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚90085799132160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚102450009562624 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚92502048454144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚90085816171520 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23113976 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21155701 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 $ _timestamp‚1.788446158384952e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚53224060 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚93836363298304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7671900 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1401841250304 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚91415873878767 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1334314844160 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53236492 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1335134272000 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 trainer/global_step‚1599 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚92558419495936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1330874142419 % _timestamp‚1.7884461590976794e+09 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚93893535764992 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21168812 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚1274.569730596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7676390 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚91470922967183 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚92558419495936 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53236492 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181260288 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1335134272000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30128564 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚90140703262720 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚129 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚102512349182464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23127992 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚53256556 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚442 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7675950 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1330219704463 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚101109633582080 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23127990 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157320192 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚92558401492992 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚101109657522176 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21168370 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53256556 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23127992 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23127990 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚93894210142720 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1335790646784 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908GîåÚ&¢ÔM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13492422 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13492420 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 trainer/global_step‚1599 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚92558419495936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1330874142419 % _timestamp‚1.7884461590976794e+09 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚93893535764992 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21168812 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚1274.569730596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7676390 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚91470922967183 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚92558419495936 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53236492 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181260288 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1335134272000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30128564 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚90140703262720 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚129 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚102512349182464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚23127992 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚53256556 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚442 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7675950 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1330219704463 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚101109633582080 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23127990 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157320192 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚92558401492992 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚101109657522176 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21168370 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53256556 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23127992 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23127990 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚93894210142720 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1335790646784 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚102512530442752 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1402715600384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53236492 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1335134272000 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 Feý¥K«‚(:À í‹æÔž¤œ€ network.sent‚ 2.737234e+06 disk.sda3.in‚0 disk.rbd0.out‚0 disk.sda2.out‚0$ disk.nvme0n1.out‚4616.9990234375 cpu‚25.311844700888326 proc.cpu.threads‚17* disk./.usagePercent‚19.988348892310697 disk.drbd2266.out‚0& disk.nvme0c0n1.out‚4616.9990234375 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0( proc.memory.availableMB‚ 236.94140625 network.recv‚517010 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.dm-1.out‚ 47.24609375 disk.drbd2265.in‚0$ memory_percent‚98.55382442474365$ disk./.usageGB‚297.8408432006836 disk.sda3.out‚ 47.24609375 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚0" proc.memory.rssMB‚ 2610.5078125 disk.nvme0c0n1.in‚0.078125! disk.dm-0.out‚4616.9990234375 disk.sda2.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda1.in‚0 disk.drbd2080.in‚0 disk.sda.out‚ 47.24609375 disk.rbd1.out‚ 6.7685546875 disk.drbd1716.in‚0) proc.memory.percent‚15.933275222778327QX2¬- _wandbruntime‚1312  _runtime‚1312‰Ç­Á & lr-AdamW/pg1‚0.0001261752454669144 & lr-AdamW/pg2‚0.0001261752454669144 trainer/global_step‚1649 % _timestamp‚1.7884461969508402e+09 _runtime‚1312.422471215 _step‚131Mq]SÌ®Æ & lr-AdamW/pg1‚0.0001261752454669144 & lr-AdamW/pg2‚0.0001261752454669144 trainer/global_step‚1649 % _timestamp‚1.7884461969508402e+09 _runtime‚1312.422471215 _step‚131ƒt@È2¯- _wandbruntime‚1312  _runtime‚1312ÖéèG,°ÄO KADeviceStatsMonitor.on_train_batch_ÁlÐJž'start/active.small_pool.current‚20036 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1375957011456 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚463 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚92831022699520 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23814776 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21791134 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚96695001046528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1375300649984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1444942685184 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13892652 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚54850912 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1445118893056 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176207872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7898480 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1370238608907 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚96695675410944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚95319700396544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23814774 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚104130519727616 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200147968 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚105575462412800 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚96695675410944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23814776 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944¡Ó”MÏ'±ÉO XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚96695675410944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23814776 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1375957011456 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚463 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚92831022699520 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23814776 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21791134 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚96695001046528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1375300649984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1444942685184 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13892652 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚54850912 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1445118893056 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176207872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7898480 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1370238608907 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚96695675410944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚95319700396544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23814774 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚104130519727616 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200147968 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚105575462412800 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0„[c[æ2²- _wandbruntime‚1313  _runtime‚1313ŠhdÙÕ&³ÏM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23828790 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54883408 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23828790 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚94257636311907 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31034554 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚94256964834591 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚95376071438336 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31034554 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _runtime‚1313.163302649 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21803301 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13900820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚451 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚133 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1376120077824 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7902481 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚95376053435392 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31054616 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54883408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚1649 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚96752847890944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21803754 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13900822 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7902932 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 % _timestamp‚1.7884461976911807e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1376776452608 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54863344 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚96752173513216 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54863344 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1371055043871 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚95376071438336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172000256 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23828792 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚96752847890944 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0_˜65i ´ÔM >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23828792 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚96752847890944 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23828790 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54883408 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23828790 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚94257636311907 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31034554 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚94256964834591 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚95376071438336 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚31034554 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _runtime‚1313.163302649 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21803301 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚13900820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚451 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚133 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1376120077824 SEDeviceStatsMonitor.on_train_batchê=áq_end/allocated_bytes.small_pool.peak‚ 863814656 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7902481 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚95376053435392 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31054616 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54883408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚1649 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚96752847890944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21803754 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13900822 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7902932 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 % _timestamp‚1.7884461976911807e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1376776452608 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚54863344 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚96752173513216 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54863344 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1371055043871 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚95376071438336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172000256 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453… ¢1ü2µ- _wandbruntime‚1313  _runtime‚1313ÊóT¶Î .loss/FRt4/assignment_loss‚4.33305310521807 0loss/FBt1/assignment_loss‚0.7508799254894252 .loss/FRt2/detection_loss‚2.517419557417593 .loss/FBt2/detection_loss‚4.220296429263221 0loss/FRt3/assignment_loss‚3.9070221160320524 1loss/SRqqt1/assignment_loss‚2.538314074277878 2loss/SRqqt4/assignment_loss‚1.9317975741987292 /loss/FRt4/detection_loss‚1.8614251571042197 /loss/FBt3/assignment_loss‚0.210762097540485 1loss/SRqqt2/detection_loss‚2.6904587421347115 0loss/SRqqt3/detection_loss‚1.906739070681074 /loss/FBt3/detection_loss‚2.9492238518354057 %loss/total_loss‚55.95181687726778 trainer/global_step‚1649 2loss/SRqqt2/assignment_loss‚2.0461798555710735 1loss/SRqqt4/detection_loss‚1.6119374046857293 % _timestamp‚1.7884461976928253e+09 _step‚134 0loss/FBt2/assignment_loss‚0.4056765290972635 0loss/FBt4/assignment_loss‚0.6628832171360651 /loss/FRt3/detection_loss‚2.2813357996179704 1loss/SRqqt1/detection_loss‚2.3525941564756283 epoch‚1 /loss/FBt4/detection_loss‚1.7617112982773808 _runtime‚ 1313.16434688 1loss/SRqqt3/assignment_loss‚2.111982233582004 /loss/FRt1/detection_loss‚2.4784169903267985 /loss/FBt1/detection_loss‚1.7164701227098704 /loss/FRt1/assignment_loss‚4.279946138029513 /loss/FRt2/assignment_loss‚4.425291430565618Üúã{Y·Ó /loss/FRt3/detection_loss‚2.2813357996179704 1loss/SRqqt1/detection_loss‚2.3525941564756283 epoch‚1 /loss/FBt4/detection_loss‚1.7617112982773808 _runtime‚ 1313.16434688 1loss/SRqqt3/assignment_loss‚2.111982233582004 /loss/FRt1/detection_loss‚2.4784169903267985 /loss/FBt1/detection_loss‚1.7164701227098704 /loss/FRt1/assignment_loss‚4.279946138029513 /loss/FRt2/assignment_loss‚4.425291430565618 .loss/FRt4/assignment_loss‚4.33305310521807 0loss/FBt1/assignment_loss‚0.7508799254894252 .loss/FRt2/detection_loss‚2.517419557417593 .loss/FBt2/detection_loss‚4.220296429263221 0loss/FRt3/assignment_loss‚3.9070221160320524 1loss/SRqqt1/assignment_loss‚2.538314074277878 2loss/SRqqt4/assignment_loss‚1.9317975741987292 /loss/FRt4/detection_loss‚1.8614251571042197 /loss/FBt3/assignment_loss‚0.210762097540485 1loss/SRqqt2/detection_loss‚2.6904587421347115 0loss/SRqqt3/detection_loss‚1.906739070681074 /loss/FBt3/detection_loss‚2.9492238518354057 %loss/total_loss‚55.95181687726778 trainer/global_step‚1649 2loss/SRqqt2/assignment_loss‚2.0461798555710735 1loss/SRqqt4/detection_loss‚1.6119374046857293 % _timestamp‚1.7884461976928253e+09 _step‚134 0loss/FBt2/assignment_loss‚0.4056765290972635 0loss/FBt4/assignment_loss‚0.6628832171360651†ôv¾“¸‚(:ˆ ü‹æÔɆý gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.841' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.1682 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¾HB³F¹‚(:» ü‹æÔ‚âªÿ network.recv‚521809 disk.drbd2265.in‚0! disk.dm-0.out‚4653.4326171875 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0# memory_percent‚98.5527753829956" proc.memory.rssMB‚ 2610.5078125 disk.sda2.in‚0 disk.sda2.out‚0 disk.sda3.out‚ 47.94921875 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125) proc.memory.percent‚15.93327522277832 disk.sda1.out‚0$ disk.nvme0n1.out‚4653.4326171875 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 network.sent‚ 2.743653e+06( disk./.usagePercent‚19.9893283771367$ disk./.usageGB‚297.8554382324219 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.dm-1.out‚ 47.94921875& disk.nvme0c0n1.out‚4653.4326171875( proc.memory.availableMB‚ 237.11328125 disk.rbd1.in‚0 disk.rbd1.out‚ 6.80078125 disk.sda.in‚0 cpu‚25.308968544473647 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda.out‚ 47.94921875 proc.cpu.threads‚17bæ9 ‘º‚(:† ‹ŒæÔÔÞœý gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.31' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.062 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0›ûŒaA»‚(:¶ ‹ŒæÔñ£Çÿ disk.sda.out‚ 48.6328125 disk.dm-1.in‚0 disk.drbd1716.in‚0# memory_percent‚98.5558271408081* disk./.usagePercent‚19.990801188481186 disk.sda3.out‚ 48.6328125 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚4707.060546875 disk.sda1.out‚0# disk.nvme0n1.out‚4707.060546875 disk.drbd2265.in‚0 network.sent‚ 2.850745e+06 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.rbd1.out‚ 7.01171875 disk.drbd2265.out‚0) proc.memory.percent‚15.93327522277832 network.recv‚526761# disk./.usageGB‚297.877384185791 disk.drbd2266.in‚0 disk.dm-0.out‚4707.060546875 disk.sda2.in‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 48.6328125( proc.memory.availableMB‚ 236.61328125 disk.sda.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2610.5078125 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 cpu‚25.306112366264426 proc.cpu.threads‚17 disk.rbd0.out‚0‘sbž¼‚(:“ šŒæÔ—«Ÿý gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚162.254' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚32.450799999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0,^þD½‚(:¹ šŒæԳƊÿ disk.rbd0.in‚0 disk.dm-1.out‚ 49.44921875( proc.memory.availableMB‚ 237.10546875 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.rbd1.in‚0 disk.sda1.in‚0 network.recv‚530979* disk./.usagePercent‚19.991939397978417# disk./.usageGB‚297.894344329834 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.drbd2266.out‚0 disk.dm-0.out‚4759.892578125 disk.sda.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 7.13671875 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda1.out‚0 disk.dm-1.in‚0 disk.drbd2265.in‚0% disk.nvme0c0n1.out‚4759.892578125$ memory_percent‚98.55282306671143) proc.memory.percent‚15.93327522277832 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125 disk.sda3.in‚0" proc.memory.rssMB‚ 2610.5078125 cpu‚25.303345227716942# disk.nvme0n1.out‚4759.892578125 disk.drbd1716.out‚0 network.sent‚ 2.85642e+06 disk.rbd0.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 49.44921875 disk.sda.out‚ 49.44921875ÕnE2¾- _wandbruntime‚1351  _runtime‚1351èå¬AÇ¿Á trainer/global_step‚1699 $ _timestamp‚1.788446235873845e+09 _runtime‚ 1351.34544182 _step‚135 ' lr-AdamW/pg1‚0.00012605284771821215 ' lr-AdamW/pg2‚0.00012605284771821215Ä•¿ ÌÀÆ _runtime‚ 1351.34544182 _step‚135 ' lr-AdamW/pg1‚0.00012605284771821215 ' lr-AdamW/pg2‚0.00012605284771821215 trainer/global_step‚1699 $ _timestamp‚1.788446235873845e+09‡ÁÛJƒ2Á- _wandbruntime‚1351  _runtime‚1351¯¾ÒiË'ÂÅO aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚107213056610816 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1416286455808 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56477764 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚99553638794752 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31942152 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14301052 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1416942817280 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 % _timestamp‚1.7884462358749332e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚56457726 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31942152 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8125941 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1416286455808 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚98137352338944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚95576212188160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚98137370341888 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚98137352338944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24515576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1487939000320 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24515574 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚107213080550912 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚95576229227520 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24515576 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22426995 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14301054 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8125487 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚108701195759104 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚1699 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 _runtime‚1351.346979803 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31962188 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚98137370341888 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176207872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1488115208192 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96987957613583 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1411728386063 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚99554313159168 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200147968 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96987286136475 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24515574 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚99553638794752 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1411073948315 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414‘â¬ñÃÊO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8125941 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1416286455808 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚98137352338944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚95576212188160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚98137370341888 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚98137352338944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24515576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1487939000320 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24515574 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚107213080550912 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚95576229227520 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24515576 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22426995 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14301054 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8125487 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚108701195759104 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 trainer/global_step‚1699 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 _runtime‚1351.346979803 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31962188 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚98137370341888 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176207872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1488115208192 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96987957613583 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1411728386063 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚99554313159168 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200147968 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96987286136475 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24515574 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚99553638794752 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1411073948315 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚107213056610816 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1416286455808 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56477764 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚99553638794752 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31942152 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14301052 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1416942817280 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 % _timestamp‚1.7884462358749332e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚56457726 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31942152ˆ-^Aj2Ä- _wandbruntime‚1352  _runtime‚135253Õ&ÅÏM VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚98193705377792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚99611485639168 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22439229 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56490196 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚95631133358080 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31980668 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1417105883648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1488962415104 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14309220 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22439674 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚97043678179315 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚98193705377792 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191746048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _step‚137 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚98193723380736 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚56510260 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56490196 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56510260 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31960606 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚108763501957632 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167805952 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚97043006701999 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚95631116318720 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚99611485639168 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8130452 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8130009 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1488794609152 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24529592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24529592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 trainer/global_step‚1699 _runtime‚1352.087518373 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚107274707348480 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31980668 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚108763693703680 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚107274731288576 % _timestamp‚1.7884462366154935e+09%,BËÚ&ÆÔM REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24529592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 trainer/global_step‚1699 _runtime‚1352.087518373 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚107274707348480 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31980668 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚108763693703680 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚107274731288576 % _timestamp‚1.7884462366154935e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚98193705377792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚99611485639168 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22439229 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56490196 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚95631133358080 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31980668 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1417105883648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1488962415104 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14309220 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22439674 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚97043678179315 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚98193705377792 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191746048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _step‚137 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚98193723380736 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚56510260 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚56490196 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56510260 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31960606 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚108763501957632 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167805952 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚97043006701999 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚95631116318720 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚99611485639168 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8130452 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8130009 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1488794609152 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24529592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0‰Y)Ø 2Ç- _wandbruntime‚1352  _runtime‚1352–A¥¨TÈÎ /loss/FRt2/assignment_loss‚4.822267416984804 /loss/FRt3/detection_loss‚2.1961826216843394 1loss/SRqqt1/assignment_loss‚2.616124451160431 1loss/SRqqt4/detection_loss‚1.9025515359685277 $ _timestamp‚1.788446236617214e+09 1loss/SRqqt3/assignment_loss‚2.047147368931252 .loss/FBt4/detection_loss‚3.675110379854838 $loss/total_loss‚61.0584263817388 /loss/FRt4/assignment_loss‚4.123942524194717 2loss/SRqqt4/assignment_loss‚1.9386511634696613 /loss/FRt2/detection_loss‚3.1643851332126127 .loss/FRt4/detection_loss‚2.230208432674408 1loss/SRqqt3/detection_loss‚1.5304730548590855 -loss/FBt3/detection_loss‚1.99013069512036 _runtime‚1352.088708088 /loss/FRt1/assignment_loss‚5.432254429786436 1loss/FBt1/assignment_loss‚0.26078819082326715 .loss/FBt2/detection_loss‚3.992625097433726 2loss/SRqqt2/assignment_loss‚1.9338192184423577 0loss/FBt4/assignment_loss‚0.6560217982284181 1loss/SRqqt2/detection_loss‚2.8391764780570723 trainer/global_step‚1699 _step‚138 /loss/FRt3/assignment_loss‚3.755796954366896 /loss/FRt1/detection_loss‚3.1018810925945157 0loss/SRqqt1/detection_loss‚2.593924636642138 /loss/FBt1/detection_loss‚3.6502760716785607 epoch‚1 1loss/FBt2/assignment_loss‚0.37134514169318006 1loss/FBt3/assignment_loss‚0.23334249387719047‰3}{YÉÓ 1loss/SRqqt4/detection_loss‚1.9025515359685277 $ _timestamp‚1.788446236617214e+09 1loss/SRqqt3/assignment_loss‚2.047147368931252 .loss/FBt4/detection_loss‚3.675110379854838 $loss/total_loss‚61.0584263817388 /loss/FRt4/assignment_loss‚4.123942524194717 2loss/SRqqt4/assignment_loss‚1.9386511634696613 /loss/FRt2/detection_loss‚3.1643851332126127 .loss/FRt4/detection_loss‚2.230208432674408 1loss/SRqqt3/detection_loss‚1.5304730548590855 -loss/FBt3/detection_loss‚1.99013069512036 _runtime‚1352.088708088 /loss/FRt1/assignment_loss‚5.432254429786436 1loss/FBt1/assignment_loss‚0.26078819082326715 .loss/FBt2/detection_loss‚3.992625097433726 2loss/SRqqt2/assignment_loss‚1.9338192184423577 0loss/FBt4/assignment_loss‚0.6560217982284181 1loss/SRqqt2/detection_loss‚2.8391764780570723 trainer/global_step‚1699 _step‚138 /loss/FRt3/assignment_loss‚3.755796954366896 /loss/FRt1/detection_loss‚3.1018810925945157 0loss/SRqqt1/detection_loss‚2.593924636642138 /loss/FBt1/detection_loss‚3.6502760716785607 epoch‚1 1loss/FBt2/assignment_loss‚0.37134514169318006 1loss/FBt3/assignment_loss‚0.23334249387719047 /loss/FRt2/assignment_loss‚4.822267416984804 /loss/FRt3/detection_loss‚2.1961826216843394 1loss/SRqqt1/assignment_loss‚2.616124451160431Š« ã’Ê‚(:‡ ©ŒæÔײý gpu.0.gpu‚47.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.695' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.139 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿·èHË‚(:½ ©ŒæÔѵƒ€ disk.sda1.out‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 network.recv‚534034 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.dm-1.in‚0! disk.dm-0.out‚4826.6513671875 disk.sda2.out‚0 disk.rbd1.in‚0" proc.memory.rssMB‚ 2610.5078125 disk.drbd2080.out‚0 disk.rbd1.out‚ 7.13671875 disk.dm-1.out‚ 49.83984375 disk.drbd2265.out‚0( proc.memory.availableMB‚ 237.32421875) proc.memory.percent‚15.93327522277832 disk.sda3.out‚ 49.83984375$ memory_percent‚98.55148792266846 disk.sda3.in‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125$ disk.nvme0n1.out‚4826.6513671875 cpu‚25.30058375329651 disk.nvme0c0n1.in‚0.078125& disk.nvme0c0n1.out‚4826.6513671875 disk.dm-0.in‚0.078125 disk.sda.out‚ 49.83984375 disk.sda1.in‚0 disk.drbd1716.in‚0 proc.cpu.threads‚17 network.sent‚ 2.861582e+06$ disk./.usageGB‚297.9034652709961 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda2.in‚0* disk./.usagePercent‚19.992551511992783î}œ žÌ‚(:“ ¸ŒæÔúöêü gpu.0.gpu‚29.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚125.542' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.108400000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0íéÆ¥KÍ‚(:À ¸ŒæÔÉÔ‘ disk.sda2.out‚0 disk.dm-1.out‚ 50.41796875 disk.drbd2265.out‚0 disk.sda3.out‚ 50.41796875 disk.sda2.in‚0 cpu‚25.297875183299023 proc.cpu.threads‚17* disk./.usagePercent‚19.991099693266108 disk.drbd2080.out‚0$ disk.nvme0n1.out‚4872.5693359375 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.rbd1.out‚ 7.1689453125# memory_percent‚98.5530138015747( proc.memory.availableMB‚ 237.07421875 disk.drbd1716.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 50.41796875 disk.rbd0.in‚0 disk.sda1.out‚0 network.recv‚540896 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125& disk.nvme0c0n1.out‚4872.5693359375 network.sent‚ 2.970208e+06 disk.sda.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0! disk.dm-0.out‚4872.5693359375" proc.memory.rssMB‚ 2610.5078125) proc.memory.percent‚15.93327522277832% disk./.usageGB‚297.88183212280273 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0ºÙ¸¶2Î- _wandbruntime‚1389  _runtime‚1389_•5ôÉÏà _step‚139 ' lr-AdamW/pg1‚0.00012591738129922247 ' lr-AdamW/pg2‚0.00012591738129922247 trainer/global_step‚1749 % _timestamp‚1.7884462742452686e+09 _runtime‚1389.716883824Â.ÎÐÈ trainer/global_step‚1749 % _timestamp‚1.7884462742452686e+09 _runtime‚1389.716883824 _step‚139 ' lr-AdamW/pg1‚0.00012591738129922247 ' lr-AdamW/pg2‚0.00012591738129922247‹"©Š•2Ñ- _wandbruntime‚1389  _runtime‚1389Ýèk3 ÒÍO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚102412276542976 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32888240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32868204 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚110295593494016 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceSPC5á tatsMonitor.on_train_batch_start/active.small_pool.freed‚32868204 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25216376 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚99773999480991 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32888240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14709454 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _step‚140 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚102412276542976 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1451909287723 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1531079217152 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1452563725471 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚1389.717940509 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25216376 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1457272261632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159430656 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚58104616 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844ôØ'ÓÒO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚102412276542976 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1451909287723 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1531079217152 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1452563725471 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚1389.717940509 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25216376 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1457272261632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159430656 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚58104616 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚102412276542976 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32888240 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32868204 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚110295593494016 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32868204 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25216376 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚99773999480991 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32888240 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14709454 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _step‚140Œ1×€M2Ô- _wandbruntime‚1390  _runtime‚1390ݨ`&Ý&Õ×M E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚102470123387392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1458091689472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1452725722687 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚102470123387392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚98376322846720 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32906720 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 202231808 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25230392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58137112 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚58117048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1458748064256 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚101011375323136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14717622 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23075201 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32886658 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚99829720046723 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58137112 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚98376339886080 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32906720 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚111889166242304 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚101011357320192 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚58117048 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚111889368474112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚110357268171776 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1531922010624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚101011357320192 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0CnTVÖÜM aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1532100302336 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚101011375323136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚58117048 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚111889368474112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚110357268171776 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1531922010624 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚101011357320192 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚102470123387392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1458091689472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1452725722687 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚102470123387392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚98376322846720 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32906720 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 202231808 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25230392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58137112 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚58117048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1458748064256 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚101011375323136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14717622 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23075201 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32886658 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚99829720046723 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚58137112 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚98376339886080 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32906720 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚111889166242304 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚101011357320192 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚418 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185467904 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25917176 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161527808 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚105270914291200 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15117854 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 trainer/global_step‚1799 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23699887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚420 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1574072062464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33814292 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59711430 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚103772656223744 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1492744627131 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚105271588655616 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1498258067456 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚101066625244160 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚103772674226688 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚113378154317312 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚102559369871291 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 $ _timestamp‚1.788446313299172e+09 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1574233590272 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1493399064879 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1498258067456 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25917174 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚418 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185467904 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25917176 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161527808 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚105270914291200 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15117854 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 trainer/global_step‚1799 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23699887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚420 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1574072062464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚33814292 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59711430 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚103772656223744 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1492744627131 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚105271588655616 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1498258067456 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚101066625244160 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚103772674226688 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚113378154317312 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚102559369871291 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 $ _timestamp‚1.788446313299172e+09 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1574233590272 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1493399064879 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414[¦2æ- _wandbruntime‚1429  _runtime‚1429D/œçÚM C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33832?êMÔD&772 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8586533 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚1799 $ _timestamp‚1.788446314026402e+09 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚103829009262592 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25931190 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚113439805054976 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1575093943296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚103829027265536 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚105328086757888 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚105328761135616 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1499077495296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25931190 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33812710 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23712111 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 _runtime‚1429.498552368 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚103829027265536 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59763964 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚444 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15126020 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚115014898998272 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚59743900 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15126022 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚115014703057920 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚113439781114880 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172000256 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1494215500051 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚102615761914131 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1493561062095 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25931192 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1499077495296 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚102615090436815 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1499733870080 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1499733870080 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8586091 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚101121546414080 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59763964 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚444 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15126020 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚115014898998272 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚59743900 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15126022 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚115014703057920 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚113439781114880 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172000256 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1494215500051 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚102615761914131 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1493561062095 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25931192 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1499077495296 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚102615090436815 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1499733870080 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1499733870080 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8586091 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚101121546414080 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚1799 $ _timestamp‚1.788446314026402e+09 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚103829009262592 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚25931190 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚113439805054976 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1575093943296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚103829027265536 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚105328086757888 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚105328761135616 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1499077495296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚25931190 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33812710 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23712111 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 _runtime‚1429.498552368 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚103829027265536 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096‘¾n“2é- _wandbruntime‚1429  _runtime‚1429*±’TêÎ 0loss/FRt4/assignment_loss‚3.6218098093723428 2loss/SRqqt2/assignment_loss‚1.7940947233754045 2loss/SRqqt3/assignment_loss‚2.3701380536872514 _runtime‚1429.499677432 /loss/FRt2/detection_loss‚2.4436249767030986 epoch‚1 .loss/FBt2/detection_loss‚3.265440359002068 /loss/FRt3/assignment_loss‚2.757297897338867 0loss/FBt2/assignment_loss‚0.3531478237307402 0loss/FBt3/assignment_loss‚0.3112192146105509 1loss/SRqqt1/detection_loss‚3.0014733877675286 1loss/SRqqt3/detection_loss‚1.9396574704014524 /loss/FRt1/assignment_loss‚4.989602658806778 /loss/FRt2/assignment_loss‚4.312620970181056 1loss/SRqqt4/assignment_loss‚1.880723061453965 0loss/FBt4/assignment_loss‚0.9408754872856662 /loss/FRt3/detection_loss‚2.1170125570562153 0loss/SRqqt4/detection_loss‚1.674265454730226 .loss/FBt3/detection_loss‚1.452461370092351 trainer/global_step‚1799 0loss/FBt1/assignment_loss‚0.3189218231045743 /loss/FRt4/detection_loss‚2.1109433153579973 %loss/total_loss‚62.29357173185126 _step‚146 .loss/FBt4/detection_loss‚2.171459913253784 % _timestamp‚1.7884463140281637e+09 /loss/FRt1/detection_loss‚2.6673604337180534 /loss/SRqqt2/detection_loss‚2.48227238129167 /loss/FBt1/detection_loss‚10.268978495697267 1loss/SRqqt1/assignment_loss‚3.048170093832345‘|YëÓ 0loss/FRt4/assignment_loss‚3.6218098093723428 2loss/SRqqt2/assignment_loss‚1.7940947233754045 2loss/SRqqt3/assignment_loss‚2.3701380536872514 _runtime‚1429.499677432 /loss/FRt2/detection_loss‚2.4436249767030986 epoch‚1 .loss/FBt2/detection_loss‚3.265440359002068 /loss/FRt3/assignment_loss‚2.757297897338867 0loss/FBt2/assignment_loss‚0.3531478237307402 0loss/FBt3/assignment_loss‚0.3112192146105509 1loss/SRqqt1/detection_loss‚3.0014733877675286 1loss/SRqqt3/detection_loss‚1.9396574704014524 /loss/FRt1/assignment_loss‚4.989602658806778 /loss/FRt2/assignment_loss‚4.312620970181056 1loss/SRqqt4/assignment_loss‚1.880723061453965 0loss/FBt4/assignment_loss‚0.9408754872856662 /loss/FRt3/detection_loss‚2.1170125570562153 0loss/SRqqt4/detection_loss‚1.674265454730226 .loss/FBt3/detection_loss‚1.452461370092351 trainer/global_step‚1799 0loss/FBt1/assignment_loss‚0.3189218231045743 /loss/FRt4/detection_loss‚2.1109433153579973 %loss/total_loss‚62.29357173185126 _step‚146 .loss/FBt4/detection_loss‚2.171459913253784 % _timestamp‚1.7884463140281637e+09 /loss/FRt1/detection_loss‚2.6673604337180534 /loss/SRqqt2/detection_loss‚2.48227238129167 /loss/FBt1/detection_loss‚10.268978495697267 1loss/SRqqt1/assignment_loss‚3.048170093832345’àÒn›“ì‚(:ˆ ôŒæÔã•Ÿý gpu.0.gpu‚46.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.839' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.1678 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÝÞ«Aí‚(:¶ ôŒæÔ·‹·ÿ( proc.memory.availableMB‚ 237.20703125 disk.rbd0.in‚0) proc.memory.percent‚15.93327522277832 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125" disk.dm-0.out‚5049.83740234375$ memory_percent‚98.55220317840576* disk./.usagePercent‚19.996486859809124' disk.nvme0c0n1.out‚5049.83740234375 disk.sda3.in‚0 disk.sda3.out‚52.4375 disk.dm-1.in‚0 disk.drbd2265.in‚0" proc.memory.rssMB‚ 2610.5078125 cpu‚25.287643173079765 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.out‚0% disk.nvme0n1.out‚5049.83740234375 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.rbd1.in‚0 disk.dm-1.out‚52.4375 disk.sda1.out‚0 network.sent‚ 3.095549e+06 network.recv‚559753 disk.dm-0.in‚0.078125 disk.sda.out‚52.4375 disk.sda1.in‚0 disk.drbd1716.out‚0 disk.drbd1716.in‚0 disk.drbd2265.out‚0$ disk./.usageGB‚297.9621047973633 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 7.388671875 disk.nvme0n1.in‚0.078125v³mÛ“î‚(:ˆ ƒæÔ ¯ý gpu.0.gpu‚36.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.872' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.1744 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚01b–ßHï‚(:½ ƒæÔ€†œÿ disk.sda3.out‚ 52.859375 disk.rbd0.in‚0* disk./.usagePercent‚19.997892341172108 disk.sda.out‚ 52.859375 disk.dm-1.out‚ 52.859375 disk.drbd2080.in‚0 network.recv‚564929 disk.drbd1716.out‚0 disk.drbd2266.in‚0$ memory_percent‚98.55368137359619) proc.memory.percent‚15.93327522277832 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125% disk./.usageGB‚297.98304748535156 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 cpu‚25.285249014215744 network.sent‚ 3.202732e+06 disk.sda2.out‚0 disk.rbd1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0( proc.memory.availableMB‚ 236.96484375" proc.memory.rssMB‚ 2610.5078125 disk.sda.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 7.513671875% disk.nvme0n1.out‚5097.17529296875 disk.drbd2080.out‚0 proc.cpu.threads‚17" disk.dm-0.out‚5097.17529296875 disk.drbd2265.out‚0 disk.sda1.out‚0 disk.dm-1.in‚0' disk.nvme0c0n1.out‚5097.17529296875WF#2ð- _wandbruntime‚1467  _runtime‚1467)ÃYÈñ ' lr-AdamW/pg1‚0.00012560735809198357 ' lr-AdamW/pg2‚0.00012560735809198357 trainer/global_step‚1849 $ _timestamp‚1.788446351910228e+09 _runtime‚1467.381850861 _step‚147ö…ÍòÇ $ _timestamp‚1.788446351910228e+09 _runtime‚1467.381850861 _step‚147 ' lr-AdamW/pg1‚0.00012560735809198357 ' lr-AdamW/pg2‚0.00012560735809198357 trainer/global_step‚1849“e ßi2ó- _wandbruntime‚1467  _runtime‚1467¹DüîôÑO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26617976 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1539243873280 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚106590326169088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1539900234752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚148 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1539900234752 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚106590308166144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1539243873280 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚103811831772160 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34740344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24336289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚116460691200512 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1617179271168 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1533579966539 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26617974 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 trainer/global_step‚1849 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34720308 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚118077846531584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚105346083215807 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195953664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚1467.382887327 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15526254 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚108130226403840 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚108129552039424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172013568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24336753 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15526252 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚118078042485248 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚103811848811520 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1617351284736 % _timestamp‚1.7884463519110396e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61338282 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚108129552039424 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8810499 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚108130226403840 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26617976 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0ý­ïÜ'õÖO JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34740344 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚462 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚118077846531584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚105346083215807 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195953664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚1467.382887327 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15526254 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚108130226403840 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚108129552039424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172013568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24336753 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15526252 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚118078042485248 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚103811848811520 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1617351284736 % _timestamp‚1.7884463519110396e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61338282 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚108129552039424 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8810499 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚108130226403840 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26617976 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26617976 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1539243873280 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚106590326169088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1539900234752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚148 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1539900234752 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚106590308166144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1539243873280 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚103811831772160 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34740344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24336289 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚116460691200512 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1617179271168 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1533579966539 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26617974 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 trainer/global_step‚1849 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34720308 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1540063301120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61370752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚1849 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚118140549621248 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34758824 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1540719675904 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1535050839459 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26631990 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34738762 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61390816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15534422 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚118140362069504 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚103866735902720 _runtime‚1468.342720028 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚116522317998080 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1618207683072 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34738762 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1534396401503 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26631992 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24349019 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚103866752942080 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚106646661204992 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚108186724506112 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚105401803781539 _step‚149 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚106646679207936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1618044071424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24349469 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 % _timestamp‚1.7884463528704715e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1540063301120 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚105401132304223 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8814599v7÷2#øàM I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1540063301120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61370752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚1849 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚118140549621248 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34758824 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1540719675904 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1535050839459 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26631990 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34738762 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61390816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15534422 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚118140362069504 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚103866735902720 _runtime‚1468.342720028 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚116522317998080 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1618207683072 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34738762 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1534396401503 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26631992 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24349019 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚103866752942080 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚106646661204992 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚108186724506112 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚105401803781539 _step‚149 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚106646679207936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1618044071424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24349469 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 % _timestamp‚1.7884463528704715e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1540063301120 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚105401132304223 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8814599 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0•¬xÙá2ù- _wandbruntime‚1468  _runtime‚1468æ–óÚRúÌ epoch‚1 /loss/FRt3/detection_loss‚2.1737865186399885 /loss/FRt4/detection_loss‚2.1127333877578613 /loss/FRt3/assignment_loss‚3.064982733461592 .loss/FRt2/detection_loss‚2.921445093197482 0loss/SRqqt1/detection_loss‚2.900084062858864 ,loss/FBt1/detection_loss‚4.4707729042747 1loss/SRqqt3/assignment_loss‚2.164654772961512 trainer/global_step‚1849 /loss/FRt1/assignment_loss‚5.277524646590738 2loss/SRqqt2/assignment_loss‚1.7854148506425147 _runtime‚1468.343836125 1loss/SRqqt4/assignment_loss‚1.670935434148167 1loss/FBt2/assignment_loss‚0.46155565426163825 /loss/FBt3/detection_loss‚1.4105080725533552 _step‚150 /loss/FRt4/assignment_loss‚4.002301971117656 1loss/FBt3/assignment_loss‚0.26756994488442304 0loss/FBt4/assignment_loss‚0.6147971451282501 .loss/FRt1/detection_loss‚3.049767772941028 0loss/SRqqt2/detection_loss‚2.272683264904244 /loss/FRt2/assignment_loss‚4.280856360707964 1loss/SRqqt3/detection_loss‚1.6467197495512664 .loss/FBt4/detection_loss‚1.888112354470286 %loss/total_loss‚56.51723195298792 $ _timestamp‚1.788446352872325e+09 1loss/SRqqt1/assignment_loss‚2.695075653217457 1loss/FBt1/assignment_loss‚0.46136946017549446 1loss/SRqqt4/detection_loss‚1.7428222212827567 /loss/FBt2/detection_loss‚3.1807579232586756¨Z7\WûÑ /loss/FRt3/assignment_loss‚3.064982733461592 .loss/FRt2/detection_loss‚2.921445093197482 0loss/SRqqt1/detection_loss‚2.900084062858864 ,loss/FBt1/detection_loss‚4.4707729042747 1loss/SRqqt3/assignment_loss‚2.164654772961512 trainer/global_step‚1849 /loss/FRt1/assignment_loss‚5.277524646590738 2loss/SRqqt2/assignment_loss‚1.7854148506425147 _runtime‚1468.343836125 1loss/SRqqt4/assignment_loss‚1.670935434148167 1loss/FBt2/assignment_loss‚0.46155565426163825 /loss/FBt3/detection_loss‚1.4105080725533552 _step‚150 /loss/FRt4/assignment_loss‚4.002301971117656 1loss/FBt3/assignment_loss‚0.26756994488442304 0loss/FBt4/assignment_loss‚0.6147971451282501 .loss/FRt1/detection_loss‚3.049767772941028 0loss/SRqqt2/detection_loss‚2.272683264904244 /loss/FRt2/assignment_loss‚4.280856360707964 1loss/SRqqt3/detection_loss‚1.6467197495512664 .loss/FBt4/detection_loss‚1.888112354470286 %loss/total_loss‚56.51723195298792 $ _timestamp‚1.788446352872325e+09 1loss/SRqqt1/assignment_loss‚2.695075653217457 1loss/FBt1/assignment_loss‚0.46136946017549446 1loss/SRqqt4/detection_loss‚1.7428222212827567 /loss/FBt2/detection_loss‚3.1807579232586756 epoch‚1 /loss/FRt3/detection_loss‚2.1737865186399885 /loss/FRt4/detection_loss‚2.1127333877578613–èós“ü‚(:ˆ ’æǪ̂©ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.082' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2164 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ï ôBý‚(:· ’æÔ‹„´€* disk./.usagePercent‚19.998772495074824 disk.sda3.out‚53.65625 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚5137.55029296875 disk.sda.out‚53.65625 disk.rbd1.out‚ 7.576171875 disk.drbd1716.in‚0$ memory_percent‚98.55623245239258" proc.memory.rssMB‚ 2610.5078125) proc.memory.percent‚15.93327522277832 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.sda1.in‚0 cpu‚25.282854322698473 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚5137.55029296875 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 network.recv‚570012 disk.drbd2265.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0" disk.dm-0.out‚5137.55029296875& proc.memory.availableMB‚ 236.546875 network.sent‚ 3.209421e+06 disk.nvme0c0n1.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚53.65625 proc.cpu.threads‚17$ disk./.usageGB‚297.9961624145508 disk.rbd0.out‚0 disk.sda1.out‚0 disk.dm-1.in‚0?a:“þ‚(:ˆ ¡æÔÖ‚“ý gpu.0.gpu‚44.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.151' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4302 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0]¶„Ç;ÿ‚(:° ¡æÔ©‰‡ÿ! disk.nvme0c0n1.out‚ 5189.28125 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.rbd0.out‚0 disk.sda.out‚ 54.29296875# proc.memory.rssMB‚ 2610.51171875 network.sent‚ 3.316665e+06$ disk./.usageGB‚298.0089340209961 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 network.recv‚575418 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.drbd2265.out‚0 disk.dm-0.out‚ 5189.28125 disk.sda.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 7.576171875 disk.nvme0n1.out‚ 5189.28125 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0) disk./.usagePercent‚19.99962960829946 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd1716.in‚0 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.sda3.out‚ 54.29296875 disk.dm-1.out‚ 54.29296875( proc.memory.availableMB‚ 236.54296875) proc.memory.percent‚15.93329906463623 cpu‚25.280759146233798 disk.dm-0.in‚0.078125$ memory_percent‚98.55625629425049)‹¯Kž€‚(:“ °æÔç²”ý gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚143.969' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.793799999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0èxJ‚(:¿ °æÔ댸€ disk.sda1.in‚0& disk.nvme0c0n1.out‚5233.7177734375 disk.sda3.out‚ 54.82421875 disk.rbd0.out‚0! disk.dm-0.out‚5233.7177734375 disk.sda.out‚ 54.82421875 disk.sda2.in‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 54.82421875 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.rbd1.out‚ 7.6708984375 disk.drbd2080.out‚0' proc.memory.availableMB‚ 236.5390625# proc.memory.rssMB‚ 2610.51171875 cpu‚25.278448590964345 network.sent‚ 3.323112e+06 network.recv‚580183 disk.sda.in‚0 disk.sda2.out‚0 disk.drbd2266.in‚0% disk./.usageGB‚298.03413009643555 disk.rbd0.in‚0 disk.drbd1716.out‚0) proc.memory.percent‚15.93329906463623 proc.cpu.threads‚17 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125) disk./.usagePercent‚20.00132053806312$ disk.nvme0n1.out‚5233.7177734375 disk.dm-1.in‚0 disk.drbd2080.in‚0# memory_percent‚98.5562801361084<]2‚- _wandbruntime‚1506  _runtime‚1506faTǃÁ trainer/global_step‚1899 % _timestamp‚1.7884463907125404e+09 _runtime‚1506.184175277 _step‚151 & lr-AdamW/pg1‚0.0001254328659121061 & lr-AdamW/pg2‚0.0001254328659121061Øù4Ì„Æ & lr-AdamW/pg2‚0.0001254328659121061 trainer/global_step‚1899 % _timestamp‚1.7884463907125404e+09 _runtime‚1506.184175277 _step‚151 & lr-AdamW/pg1‚0.0001254328659121061—Çœl2…- _wandbruntime‚1506  _runtime‚1506SG‡×'†ÑO SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9039008 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚62965134 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚442 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9039448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27318776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1660399590400 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚108132125083215 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚106557038300160 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7884463907132745e+09 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35666396 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚35666396 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚110988864152064 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚109407960108544 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24973660 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15934654 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35646360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1580229679104 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚35646360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27318776 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚110988864152064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚109407960108544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1580886040576 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195953664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24974102 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚119543204143616 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚110988189787648 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚121203627674112 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚119543228083712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚1506.185147822 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1575069743695 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 trainer/global_step‚1899 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚108131453606107 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1580229679104 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚121203431720448 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚106557055339520 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1660227576832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚110988189787648 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚62985172 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27318774 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172013568 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚110988189787648 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚62985172 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27318774 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172013568 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195953664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24974102 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚119543204143616 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚110988189787648 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚121203627674112 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚119543228083712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚1506.185147822 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1575069743695 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 trainer/global_step‚1899 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚108131453606107 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1580229679104 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚121203431720448 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚106557055339520 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1660227576832˜Eh f2ˆ- _wandbruntime‚1506  _runtime‚1506¤‘WO_ ‰ÛM B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚63017668 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1581705481728 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚1899 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27332792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚111046036632064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63017668 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚121266127148544 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1581049106944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚111045362254336 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35664814 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15942822 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚108187174171631 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1575231740911 % _timestamp‚1.7884463914290044e+09 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚462 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚106611942430720 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27332792 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚121265933305344 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize^b‚_allocations.current‚0 _runtime‚1506.901203723 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27332790 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚109464331150336 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚108187845648947 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚111046036632064 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24986389 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35664814 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚106611959470080 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚109464331150336 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193843200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35684876 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚109464313147392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1581049106944 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1575886178867 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚109464313147392 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1661078424064 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62997604 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0]ºc»æ&ŠàM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚27332792 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚121265933305344 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚1506.901203723 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27332790 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚109464331150336 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚108187845648947 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚111046036632064 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24986389 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚35664814 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚106611959470080 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚109464331150336 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193843200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35684876 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚109464313147392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1581049106944 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1575886178867 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚109464313147392 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1661078424064 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62997604 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚63017668 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1581705481728 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚1899 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27332792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚111046036632064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63017668 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚121266127148544 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1581049106944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚111045362254336 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35664814 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15942822 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚108187174171631 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1575231740911 % _timestamp‚1.7884463914290044e+09 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚462 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚106611942430720 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728™1¹,2‹- _wandbruntime‚1506  _runtime‚1506ëñªoUŒÏ /loss/FRt1/assignment_loss‚4.897139469782512 /loss/FRt3/detection_loss‚1.7402374186253144 /loss/FRt4/assignment_loss‚4.066328148631489 2loss/SRqqt1/assignment_loss‚2.6696122309257246 -loss/FRt1/detection_loss‚3.21363418896993 trainer/global_step‚1899 _step‚154 1loss/SRqqt3/detection_loss‚1.4453299608135568 /loss/FBt4/detection_loss‚1.1214697317960258 /loss/FRt2/assignment_loss‚4.348721894621849 0loss/FBt1/assignment_loss‚0.6363485188477777 1loss/FBt3/assignment_loss‚0.30213933127496057 0loss/FBt4/assignment_loss‚0.7727495187095234 /loss/FRt2/detection_loss‚2.4151604741811754 /loss/FBt1/detection_loss‚3.4765057216089557 epoch‚1 /loss/FRt3/assignment_loss‚2.452524305400202 2loss/SRqqt4/assignment_loss‚1.8122494094073773 0loss/FBt2/assignment_loss‚0.6980741090292213 .loss/FRt4/detection_loss‚1.980333967243924 0loss/SRqqt4/detection_loss‚1.690875424870423 .loss/FBt2/detection_loss‚2.753472829858462 1loss/SRqqt2/detection_loss‚2.2230043040381537 &loss/total_loss‚52.499794474988306 % _timestamp‚1.7884463914306946e+09 0loss/SRqqt2/assignment_loss‚2.01804896791776 2loss/SRqqt3/assignment_loss‚2.0818369676885395 1loss/SRqqt1/detection_loss‚2.6881598193069984 /loss/FBt3/detection_loss‚0.9958377614384517 _runtime‚ 1506.90217172ß¿BªZÔ 0loss/SRqqt2/assignment_loss‚2.01804896791776 2loss/SRqqt3/assignment_loss‚2.0818369676885395 1loss/SRqqt1/detection_loss‚2.6881598193069984 /loss/FBt3/detection_loss‚0.9958377614384517 _runtime‚ 1506.90217172 /loss/FRt1/assignment_loss‚4.897139469782512 /loss/FRt3/detection_loss‚1.7402374186253144 /loss/FRt4/assignment_loss‚4.066328148631489 2loss/SRqqt1/assignment_loss‚2.6696122309257246 -loss/FRt1/detection_loss‚3.21363418896993 trainer/global_step‚1899 _step‚154 1loss/SRqqt3/detection_loss‚1.4453299608135568 /loss/FBt4/detection_loss‚1.1214697317960258 /loss/FRt2/assignment_loss‚4.348721894621849 0loss/FBt1/assignment_loss‚0.6363485188477777 1loss/FBt3/assignment_loss‚0.30213933127496057 0loss/FBt4/assignment_loss‚0.7727495187095234 /loss/FRt2/detection_loss‚2.4151604741811754 /loss/FBt1/detection_loss‚3.4765057216089557 epoch‚1 /loss/FRt3/assignment_loss‚2.452524305400202 2loss/SRqqt4/assignment_loss‚1.8122494094073773 0loss/FBt2/assignment_loss‚0.6980741090292213 .loss/FRt4/detection_loss‚1.980333967243924 0loss/SRqqt4/detection_loss‚1.690875424870423 .loss/FBt2/detection_loss‚2.753472829858462 1loss/SRqqt2/detection_loss‚2.2230043040381537 &loss/total_loss‚52.499794474988306 % _timestamp‚1.7884463914306946e+09šÿ7צ“Ž‚(:ˆ ¿æÔ”í“ý gpu.0.gpu‚41.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.509' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.7018 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0##áF‚(:» ¿æÔ¼¾‘€ disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 7.7958984375 disk.dm-1.out‚ 55.3359375* disk./.usagePercent‚20.002520701383634 disk.sda.in‚0 disk.sda1.in‚0$ disk./.usageGB‚298.0520133972168 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.drbd2265.in‚0( proc.memory.availableMB‚ 236.50390625 disk.rbd0.out‚0 disk.dm-0.out‚5314.978515625 disk.drbd2265.out‚0$ memory_percent‚98.55649471282959) proc.memory.percent‚15.93329906463623 network.sent‚ 3.328359e+06 disk.drbd2080.in‚0% disk.nvme0c0n1.out‚5314.978515625 disk.drbd2266.out‚0 disk.sda2.out‚0# proc.memory.rssMB‚ 2610.51171875 cpu‚25.276239472989285 network.recv‚583748 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda.out‚ 55.3359375 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125# disk.nvme0n1.out‚5314.978515625 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.sda3.out‚ 55.3359375 disk.dm-1.in‚0 disk.drbd1716.out‚0 proc.cpu.threads‚17áÈ‚“‚(:ˆ ÎæÔ´§ýü gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.082' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.0164 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0±‹¯g>‘‚(:³ ÎæÔÐО€* disk./.usagePercent‚20.003102606508964 disk.rbd1.in‚0 disk.rbd1.out‚ 7.8583984375 disk.dm-1.out‚ 56.1640625 disk.drbd2080.out‚0# proc.memory.rssMB‚ 2610.51171875) proc.memory.percent‚15.93329906463623 disk.drbd2266.out‚0 disk.nvme0n1.in‚0.078125# disk.nvme0n1.out‚5355.392578125 disk.dm-1.in‚0 disk.drbd2265.out‚0 disk.sda3.out‚ 56.1640625# memory_percent‚98.5565185546875 disk.drbd1716.out‚0 disk.dm-0.out‚5355.392578125 disk.sda.out‚ 56.1640625 disk.sda1.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0! proc.memory.availableMB‚236.5 disk.drbd2266.in‚0% disk./.usageGB‚298.06068420410156 disk.sda2.in‚0 disk.sda2.out‚0% disk.nvme0c0n1.out‚5355.392578125 disk.rbd0.out‚0 network.recv‚591151 disk.drbd1716.in‚0 network.sent‚ 3.437257e+06 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 proc.cpu.threads‚17 disk.drbd2265.in‚0 disk.sda1.out‚0 cpu‚25.27405962218248^WìÈ2’- _wandbruntime‚1544  _runtime‚15446›Æ“À trainer/global_step‚1949 $ _timestamp‚1.788446428747751e+09 _runtime‚1544.219316899 _step‚155 & lr-AdamW/pg1‚0.0001252454342652203 & lr-AdamW/pg2‚0.0001252454342652203hnH˔Š& lr-AdamW/pg2‚0.0001252454342652203 trainer/global_step‚1949 $ _timestamp‚1.788446428747751e+09 _runtime‚1544.219316899 _step‚155 & lr-AdamW/pg1‚0.0001252454342652203›¥–†Ò2•- _wandbruntime‚1544  _runtime‚1544Äæ߆¶&–ÑO E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64591986 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚112225612050944 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚113846827535872 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1621215484928 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚36572412 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚113846827535872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28019576 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚124329030050304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚64612024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28019574 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚122625741026816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚110918166950623 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚109302244828160 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚1544.220242976 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚467 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9268446 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191759360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28019576 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25611965 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9268911 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚124329221809664 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚109302261867520 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚122625764966912 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1703289023488 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7884464287484024e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36592448 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚113847501900288 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16343054 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚112225612050944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28019574 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1621871846400 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1621871846400 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167819264 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚110917495473515 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1621215484928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36572412 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1703456842752 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚112225630053888 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚113847501900288 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _step‚156 M@DeviceStÒf-!atsMonitor.on_train_batch_start/inactive_split.all.freed‚25611498 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16343052 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844œîÂ×Ü'—ÖO ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚113846827535872 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1621215484928 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚36572412 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚113846827535872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28019576 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚124329030050304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚64612024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28019574 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚122625741026816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚110918166950623 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚109302244828160 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚1544.220242976 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚467 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9268446 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191759360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28019576 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25611965 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9268911 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚124329221809664 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚109302261867520 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚122625764966912 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1703289023488 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7884464287484024e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36592448 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚113847501900288 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16343054 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚112225612050944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28019574 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1621871846400 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1621871846400 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167819264 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚110917495473515 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1621215484928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36572412 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1703456842752 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚112225630053888 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚113847501900288 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _step‚156 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25611498 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16343052 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64591986 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚112225612050944 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884464294516716e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 _runtime‚1544.923869857 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28033592 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚110973887516355 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚112281983092736 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36590866 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155223040 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚113904674380288 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64644520 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1622691287552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16351220 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1704323274240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚122687391764480 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚109357165998080 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9273046 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28033590 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1622691287552 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _step‚157 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚113904674380288 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚466 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 trainer/global_step‚1949 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9273510 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚64624456 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚64624456 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚109357148958720 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚112281965089792 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25624266ÓÉ"æ&šàM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1622691287552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16351220 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1704323274240 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚122687391764480 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚109357165998080 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9273046 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28033590 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1622691287552 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _step‚157 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚113904674380288 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚466 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 trainer/global_step‚1949 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9273510 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚64624456 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚64624456 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚109357148958720 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚112281965089792 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25624266 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884464294516716e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 _runtime‚1544.923869857 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28033592 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚110973887516355 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚112281983092736 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36590866 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155223040 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚113904674380288 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64644520 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0SSæ2›- _wandbruntime‚1544  _runtime‚1544\éUœÏ 0loss/SRqqt1/detection_loss‚2.528786450624466 1loss/SRqqt3/detection_loss‚1.7985258616114919 /loss/FRt4/assignment_loss‚3.879839924036288 .loss/FRt4/detection_loss‚2.471924446961459 /loss/FBt2/detection_loss‚2.3679618885119758 /loss/FBt3/detection_loss‚1.0293113922850154 .loss/FRt3/assignment_loss‚3.07569570094347 /loss/FBt4/detection_loss‚2.1428789297739663 % _timestamp‚1.7884464294533758e+09 /loss/FRt1/assignment_loss‚5.141920620395291 0loss/FBt2/assignment_loss‚0.4449417252317798 0loss/FBt3/assignment_loss‚0.2726663789344304 /loss/FRt1/detection_loss‚2.9912867834491115 %loss/total_loss‚53.85643577375026 trainer/global_step‚1949 _step‚158 /loss/FBt1/detection_loss‚2.7375291883945465 0loss/FRt2/assignment_loss‚4.1129997064327375 1loss/SRqqt1/assignment_loss‚2.900218052789569 2loss/SRqqt4/assignment_loss‚1.9914588905645139 0loss/FBt1/assignment_loss‚0.4600001532963688 2loss/SRqqt3/assignment_loss‚1.8687389924991549 _runtime‚1544.924930047 0loss/SRqqt2/detection_loss‚1.860350384660389 1loss/SRqqt4/detection_loss‚1.7799536600483186 epoch‚1 2loss/SRqqt2/assignment_loss‚2.3281763221906577 0loss/FBt4/assignment_loss‚0.4267357873937322 .loss/FRt2/detection_loss‚3.365978944918205 .loss/FRt3/detection_loss‚1.878555587803324TÄZKÔ /loss/FRt4/assignment_loss‚3.879839924036288 .loss/FRt4/detection_loss‚2.471924446961459 /loss/FBt2/detection_loss‚2.3679618885119758 /loss/FBt3/detection_loss‚1.0293113922850154 .loss/FRt3/assignment_loss‚3.07569570094347 /loss/FBt4/detection_loss‚2.1428789297739663 % _timestamp‚1.7884464294533758e+09 /loss/FRt1/assignment_loss‚5.141920620395291 0loss/FBt2/assignment_loss‚0.4449417252317798 0loss/FBt3/assignment_loss‚0.2726663789344304 /loss/FRt1/detection_loss‚2.9912867834491115 %loss/total_loss‚53.85643577375026 trainer/global_step‚1949 _step‚158 /loss/FBt1/detection_loss‚2.7375291883945465 0loss/FRt2/assignment_loss‚4.1129997064327375 1loss/SRqqt1/assignment_loss‚2.900218052789569 2loss/SRqqt4/assignment_loss‚1.9914588905645139 0loss/FBt1/assignme pÄžnt_loss‚0.4600001532963688 2loss/SRqqt3/assignment_loss‚1.8687389924991549 _runtime‚1544.924930047 0loss/SRqqt2/detection_loss‚1.860350384660389 1loss/SRqqt4/detection_loss‚1.7799536600483186 epoch‚1 2loss/SRqqt2/assignment_loss‚2.3281763221906577 0loss/FBt4/assignment_loss‚0.4267357873937322 .loss/FRt2/detection_loss‚3.365978944918205 .loss/FRt3/detection_loss‚1.878555587803324 0loss/SRqqt1/detection_loss‚2.528786450624466 1loss/SRqqt3/detection_loss‚1.7985258616114919žµº+Sžž‚(:“ ÝæÔ“’Ùý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚131.907' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.381400000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0zqh[=Ÿ‚(:² ÝæÔûÒâÿ disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 56.65234375 disk.sda3.in‚0# memory_percent‚98.5565185546875 cpu‚25.27194029681198 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0" disk.nvme0n1.out‚ 5406.16796875! proc.memory.availableMB‚236.5) disk./.usagePercent‚20.00221349234255 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.sda2.in‚0# proc.memory.rssMB‚ 2610.51171875 proc.cpu.threads‚17% disk./.usageGB‚298.04743576049805 disk.sda3.out‚ 56.65234375 disk.rbd0.out‚0) proc.memory.percent‚15.93329906463623 network.sent‚ 3.442497e+06 disk.nvme0c0n1.in‚0.078125 disk.rbd1.out‚ 8.0302734375 disk.dm-1.in‚0$ disk.nvme0c0n1.out‚ 5406.16796875 disk.rbd0.in‚0 disk.sda1.out‚0 network.recv‚594701 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125 disk.dm-0.out‚ 5406.16796875 disk.sda2.out‚0 disk.rbd1.in‚0 disk.sda.in‚0 disk.sda.out‚ 56.65234375ETÝ¥“ ‚(:ˆ ìæÔ¿áŒý gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚135.044' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚27.0088 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0­H˜‹=¡‚(:² ìæÔâ·£ÿ disk.drbd1716.out‚0 disk.drbd2080.out‚0" disk.nvme0c0n1.out‚ 5452.828125 disk.dm-0.out‚ 5452.828125# memory_percent‚98.5569953918457 disk.sda1.in‚0 disk.dm-1.in‚0 disk.drbd1716.in‚0 disk.dm-0.in‚0.078125* disk./.usagePercent‚20.003260307150057 disk.sda2.out‚0 disk.drbd2266.out‚0 network.recv‚602002$ disk./.usageGB‚298.0630340576172 disk.sda.out‚ 57.25390625 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 cpu‚25.269704114794955 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.sda.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0& proc.memory.availableMB‚ 236.421875 disk.sda1.out‚0 disk.rbd1.in‚0 disk.sda3.in‚0 disk.sda3.out‚ 57.25390625# proc.memory.rssMB‚ 2610.51171875 disk.dm-1.out‚ 57.25390625 disk.rbd0.in‚0) proc.memory.percent‚15.93329906463623 network.sent‚ 3.551005e+06 disk.rbd1.out‚ 8.1865234375 disk.nvme0n1.out‚ 5452.828125„Ãò]“¢‚(:ˆ ûæÔ™‰úü gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.233' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.2466 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0CgùC£‚(:¸ ûæÔÚïþ* disk./.usagePercent‚19.999974450448082 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-1.out‚57.8125$ memory_percent‚98.55296611785889# proc.memory.rssMB‚ 2610.51171875 proc.cpu.threads‚17$ disk./.usageGB‚298.0140724182129 disk.sda3.out‚57.8125 disk.rbd0.out‚0 network.recv‚605343 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 8.3115234375 disk.drbd1716.out‚0 disk.drbd2265.in‚0 cpu‚25.267819181689067 network.sent‚ 3.556144e+06 disk.drbd2080.out‚0 disk.sda1.out‚0 disk.drbd2265.out‚0) proc.memory.percent‚15.93329906463623 disk.drbd2266.out‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0% disk.nvme0n1.out‚5511.84619140625 disk.nvme0c0n1.in‚0.078125" disk.dm-0.out‚5511.84619140625 disk.sda.out‚57.8125 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚5511.84619140625 disk.sda3.in‚0 disk.dm-0.in‚0.078125 disk.dm-1.in‚0( proc.memory.availableMB‚ 237.08203125!½1ï2¤- _wandbruntime‚1581  _runtime‚1581ï­lɥà _step‚159 ' lr-AdamW/pg1‚0.00012504510221180055 ' lr-AdamW/pg2‚0.00012504510221180055 trainer/global_step‚1999 % _timestamp‚1.7884464661862457e+09 _runtime‚1581.657859983 öIÎ¦È ' lr-AdamW/pg1‚0.00012504510221180055 ' lr-AdamW/pg2‚0.00012504510221180055 trainer/global_step‚1999 % _timestamp‚1.7884464661862457e+09 _runtime‚1581.657859983 _step‚159ŸUʨ¥2§- _wandbruntime‚1581  _runtime‚1581šßßI×'¨ÑO JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚112047468395520 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1662201290752 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9498353 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚115043263993344 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚116706139648512 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9498795 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 _step‚160 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1662201290752 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚37498464 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚116705465284096 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚115043281996288 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1656740422511 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66238876 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1746451267584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37498464 trainer/global_step‚1999 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚1581.658880589 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28720374 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193856512 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚115043281996288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚113703537340923 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1746621184000 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26249805 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚127454923034112 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169916416 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚113704208818031 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1662857652224 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28720376 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37518500 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16751454 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚125708301850112 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚66218838 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16751452 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚116705465284096 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚127454729177600 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28720374 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26250249 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚116706139648512 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚127454729177600 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28720374 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26250249 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚116706139648512 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚116705465284096 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚115043281996288 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1656740422511 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66238876 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1746451267584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37498464 trainer/global_step‚1999 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚1581.658880589 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28720374 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193856512 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚115043281996288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚113703537340923 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1746621184000 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26249805 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚127454923034112 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169916416 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚113704208818031 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1662857652224 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28720376 §"Dí2ª- _wandbruntime‚1582  _runtime‚1582™£'«ÛM B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66271372 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16759622 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1657556857683 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1663677093376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1663677093376 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚115099635035136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚115099617032192 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28734390 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚127517431830016 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚1999 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚127517242181120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37516918 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26263001 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚452 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚125769952587776 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26262549 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1656902419727 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚113759257906447 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7884464669071782e+09 _step‚161 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9503379 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚125769928647680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/re®ýÁº quested_bytes.all.peak‚ 9643699017 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚116763312128512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1663020718592 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28734392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165708800 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115099617032192 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚66251308 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9502929 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 _runtime‚1582.379373098 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66271372 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16759620 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928Èûêæ&¬àM C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1657556857683 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1663677093376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1663677093376 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚115099635035136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚115099617032192 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28734390 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚127517431830016 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚1999 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚127517242181120 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37516918 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26263001 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚452 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚125769952587776 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26262549 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1656902419727 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚113759257906447 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7884464669071782e+09 _step‚161 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9503379 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚125769928647680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚116763312128512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1663020718592 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28734392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165708800 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115099617032192 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚66251308 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9502929 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 _runtime‚1582.379373098 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66271372 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16759620 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66271372 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16759622 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414¡\ã.÷2­- _wandbruntime‚1582  _runtime‚1582© ÖºO®É 0loss/FBt1/assignment_loss‚0.7607669033449687 /loss/FRt3/detection_loss‚1.9657206677161811 0loss/SRqqt2/detection_loss‚2.019869910898032 /loss/FRt1/assignment_loss‚5.079178457076733 /loss/FRt2/assignment_loss‚4.478608780437046 2loss/SRqqt3/assignment_loss‚1.7318117960932708 /loss/FRt2/detection_loss‚2.2991918987698026 % _timestamp‚1.7884464669088826e+09 0loss/FRt3/assignment_loss‚2.9176029970771387 /loss/FBt3/detection_loss‚0.9093571820855141 -loss/FRt4/assignment_loss‚3.8688160083333 1loss/SRqqt1/assignment_loss‚2.413489202658335 /loss/FBt4/assignment_loss‚0.668023352821668 /loss/FRt1/detection_loss‚3.1562987841092625 .loss/FRt4/detection_loss‚2.077407100650131 0loss/SRqqt4/detection_loss‚1.487988670537998 .loss/FBt4/detection_loss‚1.904468576113383 _step‚162 0loss/SRqqt3/detection_loss‚1.560804707899926 .loss/FBt1/detection_loss‚4.304140186309814 /loss/FBt2/detection_loss‚2.5552492209456186 trainer/global_step‚1999 0loss/FBt2/assignment_loss‚0.5017584300297089 0loss/FBt3/assignment_loss‚0.3534385228318466 _runtime‚1582.380381381 0loss/SRqqt1/detection_loss‚3.105955262978872 %loss/total_loss‚54.45423989802276 epoch‚1 1loss/SRqqt2/assignment_loss‚2.272598493706297 1loss/SRqqt4/assignment_loss‚2.061694784597917<ÊâT¯Î -loss/FRt4/assignment_loss‚3.8688160083333 1loss/SRqqt1/assignment_loss‚2.413489202658335 /loss/FBt4/assignment_loss‚0.668023352821668 /loss/FRt1/detection_loss‚3.1562987841092625 .loss/FRt4/detection_loss‚2.077407100650131 0loss/SRqqt4/detection_loss‚1.487988670537998 .loss/FBt4/detection_loss‚1.904468576113383 _step‚162 0loss/SRqqt3/detection_loss‚1.560804707899926 .loss/FBt1/detection_loss‚4.304140186309814 /loss/FBt2/detection_loss‚2.5552492209456186 trainer/global_step‚1999 0loss/FBt2/assignment_loss‚0.5017584300297089 0loss/FBt3/assignment_loss‚0.3534385228318466 _runtime‚1582.380381381 0loss/SRqqt1/detection_loss‚3.105955262978872 %loss/total_loss‚54.45423989802276 epoch‚1 1loss/SRqqt2/assignment_loss‚2.272598493706297 1loss/SRqqt4/assignment_loss‚2.061694784597917 0loss/FBt1/assignment_loss‚0.7607669033449687 /loss/FRt3/detection_loss‚1.9657206677161811 0loss/SRqqt2/detection_loss‚2.019869910898032 /loss/FRt1/assignment_loss‚5.079178457076733 /loss/FRt2/assignment_loss‚4.478608780437046 2loss/SRqqt3/assignment_loss‚1.7318117960932708 /loss/FRt2/detection_loss‚2.2991918987698026 % _timestamp‚1.7884464669088826e+09 0loss/FRt3/assignment_loss‚2.9176029970771387 /loss/FBt3/detection_loss‚0.9093571820855141¢Á}Ó°‚(:ˆ ŠŽæÔñБý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚172.441' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.4882 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ó7:0?±‚(:´ ŠŽæÔª úÿ disk.drbd2080.in‚0 disk.sda3.in‚0 network.sent‚ 3.562788e+06% disk./.usageGB‚298.06068420410156 disk.rbd1.out‚ 8.3740234375% disk.nvme0n1.out‚5557.42041015625 disk.drbd1716.in‚0 network.recv‚610538 disk.sda3.out‚58.5625 disk.drbd2266.out‚0 disk.dm-1.out‚58.5625% proc.memory.availableMB‚ 236.78125 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0) proc.memory.percent‚15.93334674835205 disk.rbd0.in‚0* disk./.usagePercent‚20.003102606508964 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚5557.42041015625 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.sda.in‚0 disk.sda.out‚58.5625 disk.sda1.in‚0 disk.sda2.out‚0# proc.memory.rssMB‚ 2610.51953125 cpu‚25.2657664916819 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0$ memory_percent‚98.55480194091797 disk.drbd2266.in‚0 disk.rbd0.out‚0" disk.dm-0.out‚5557.42041015625Dd·P“²‚(:ˆ ™ŽæÔ“Åöü gpu.0.gpu‚39.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.509' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.7018 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚011DZL³‚(:Á ™ŽæÔʸçÿ disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125% disk./.usageGB‚298.07151794433594 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 disk.rbd1.in‚0 network.sent‚ 3.66995e+06 network.recv‚615859 disk.dm-1.out‚ 59.2109375' disk.nvme0c0n1.out‚5616.91064453125 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 8.3740234375$ memory_percent‚98.55482578277588* disk./.usagePercent‚20.003829667906206 disk.sda3.out‚ 59.2109375 disk.rbd0.out‚0# proc.memory.rssMB‚ 2610.51953125) proc.memory.percent‚15.93334674835205 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17 disk.dm-1.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0" disk.dm-0.out‚5616.91064453125 disk.sda.in‚0 disk.sda.out‚ 59.2109375 cpu‚25.263637611150077 disk.drbd1716.in‚0 disk.rbd0.in‚0% disk.nvme0n1.out‚5616.91064453125( proc.memory.availableMB‚ 236.77734375´:¦2´- _wandbruntime‚1618  _runtime‚1618›É=€ÇµÁ _runtime‚1618.985049285 _step‚163 & lr-AdamW/pg1‚0.0001248319115007465 & lr-AdamW/pg2‚0.0001248319115007465 trainer/global_step‚2049 % _timestamp‚1.7884465035134623e+09=¡ßÌ¶Æ & lr-AdamW/pg1‚0.0001248319115007465 & lr-AdamW/pg2‚0.0001248319115007465 trainer/global_step‚2049 % _timestamp‚1.7884465035134623e+09 _runtime‚1618.985049285 _step‚163£Àa£ì2·- _wandbruntime‚1618  _runtime‚1618krw—×'¸ÑO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29421176 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 trainer/global_step‚2049 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚67845690 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29421174 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38444552 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177079296 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1789577584128 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38444552 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26888238 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚119564103032320 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67865728 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29421176 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚117860933938688 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚130580569456640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _runtime‚1618.986139142 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚130580392377344 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1703187096576 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17159854 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1703843458048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 153139200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚116490250685439 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38424516 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚114792657884160 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844’ £Õ£ ¹ÖO ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚119564103032320 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17159852 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚117860915935744 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9728384 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1789730723328 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _step‚164 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚119564777396736 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1696921324171 % _timestamp‚1.7884465035142498e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1703187096576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚29421174 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38444552 VHDeviceStatsMoná–n9itor.on_train_batch_start/inactive_split_bytes.all.current‚ 177079296 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1789577584128 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38444552 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26888238 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚119564103032320 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67865728 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29421176 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚117860933938688 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚130580569456640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _runtime‚1618.986139142 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚130580392377344 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1703187096576 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17159854 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1703843458048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 153139200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚116490250685439 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38424516 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚114792657884160 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29421176 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 trainer/global_step‚2049 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚67845690 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚117917268974592 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29435192 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26900402 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195940352 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚128852465530880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67878160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚117917286977536 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚2049 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1704662899200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17168020 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚114847579054080 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚165 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚117917286977536 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38442970 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26900854 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67898224 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29435192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1697737759135 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172000256 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38463032 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚119621949876736 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚119621275499008 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚116545299773855 _runtime‚1619.694513318 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚119621949876736 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚116545971251171 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚128852465530880 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚67878160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚117917286977536 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚2049 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1704662899200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17168020 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚114847579054080 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚165 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚117917286977536 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38442970 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26900854 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67898224 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29435192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1697737759135 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 172000256 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38463032 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚119621949876736 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚119621275499008 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚116545299773855 _runtime‚1619.694513318 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚119621949876736 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚116545971251171 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚117917268974592 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29435192 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26900402 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195940352 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚119275621075739 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚117537881451520 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30121976 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39370604 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1744829263872 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30121974 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1744172902400 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚131873375616512 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 _step‚168 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1832696955392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚1655.384168365 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1744829263872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚119276292552847 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚117537864412160 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17568252 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9957688 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1832864774656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚120678585881088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚120678567878144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9958143 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚69472542 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚133706240391168 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚131873351676416 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚122422740780544 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚2099 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚133706048631808 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167819264 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚120678567878144 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39350568 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚122423415144960 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17568254 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚122422740780544 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 # _timestamp‚1.78844653991229e+09 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39370604 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚122423415144960 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30121974 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1744172902400 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39350568 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚120678585881088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27525940 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1737756663579v¦ÁæÚ'ËÔO TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30121976 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚122422740780544 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 # _timestamp‚1.78844653991229e+09 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39370604 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚122423415144960 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30121974 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1744172902400 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39350568 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚120678585881088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27525940 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1737756663579 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27526397 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚119275621075739 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚117537881451520 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30121976 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39370604 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1744829263872 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30121974 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1744172902400 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚131873375616512 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 _step‚168 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1832696955392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _runtime‚1655.384168365 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1744829263872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚119276292552847 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚117537864412160 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17568252 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9957688 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1832864774656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚120678585881088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚120678567878144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9958143 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚69472542 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚133706240391168 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚131873351676416 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚122422740780544 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚2099 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚133706048631808 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167819264 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚120678567878144 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚39350568 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚122423415144960 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17568254 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30135992 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69525076 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚131935026354176 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚119332013118579 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1745648705024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551744 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1744992330240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30135990 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚120734920916992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚2099 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884465406122706e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚120734938919936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1745648705024 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39389084 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27539128 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30135990 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27538664 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17576420 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚462 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1833720386560 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1744992330240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚133768559188992Oø4}Þ‰°þå&ÎàM B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚117592785582080 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39369022 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚69525076 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69505012 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚133768746740736 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39389084 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30135992 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27538664 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17576420 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚462 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1833720386560 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1744992330240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚133768559188992 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚122479913247232 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚120734938919936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚464 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17576422 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚169 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30135992 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69525076 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚131935026354176 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚119332013118579 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1745648705024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551744 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1744992330240 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30135990 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚120734920916992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚2099 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884465406122706e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚120734938919936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1745648705024 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39389084 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27539128 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30135990 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1875868903936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚2149 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚125281378528768 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17976652 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚123496219820544 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40296656 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚134955888559616 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40276620 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚120283070940160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30822774 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1785815069696 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28164917 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚71119432 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1875868903936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚2149 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚125281378528768 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17976652 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚123496219820544 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40296656 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚134955888559616 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40276620 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚120283070940160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30822774 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1785815069696 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28164917 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚71119432 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28177634 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30836790 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40315136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚462 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1779408437951 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚123552572859392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚125339225373184 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚123552590862336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1786634510848 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚2149 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71131864 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1785978136064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚1692.800100684 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚125339225373184 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71131864 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40295074 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1876884576256 % _timestamp‚1.7884465773279345e+09 _step‚173 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚135017539297280 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28177170 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚71151928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17984822 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚136894247678976 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚120337992110080 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1780062875907 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚120337975070720 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1785978136064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200134656 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚123552572859392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71151928 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40295074 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10192350 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30836790 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1876708381696 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40315136 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚122118054985987 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚122117383508671 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30836792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚464 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚136894447813632 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844"Å~öè&ÞâM SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71131864 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40295074 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1876884576256 % _timestamp‚1.7884465773279345e+09 _step‚173 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚135017539297280 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28177170 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚71151928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17984822 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚136894247678976 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚120337992110080 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1780062875907 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚120337975070720 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1785978136064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200134656 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚123552572859392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71151928 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚40295074 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10192350 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30836790 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1876708381696 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40315136 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚122118054985987 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚122117383508671 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30836792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚464 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚136894447813632 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚123552590862336 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30836792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10192812 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17984820 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚135017563237376 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 176194560 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1786634510848 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28177634 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30836790 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40315136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚462 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1779408437951 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚123552572859392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚125339225373184 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚123552590862336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1786634510848 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚2149 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71131864 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1785978136064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚1692.800100684 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚125339225373184 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0­ Íò2ß- _wandbruntime‚1692  _runtime‚1692A`ª–WàÑ $ _timestamp‚1.788446577329893e+09 _runtime‚1692.801410266 /loss/FRt4/detection_loss‚2.1207254560755904 .loss/FRt2/detection_loss‚2.590565818387109 1loss/SRqqt4/detection_loss‚1.9335164735093713 .loss/FBt3/detection_loss‚1.312135426007444 .loss/FBt4/detection_loss‚2.284528374671936 /loss/FRt4/assignment_loss‚3.945579570824983 2loss/SRqqt3/assignment_loss‚1.7375876899522085 2loss/SRqqt4/assignment_loss‚1.8368736336706206 .loss/FRt1/detection_loss‚4.021948225796223 /loss/FRt3/detection_loss‚2.1394248094823625 1loss/SRqqt1/detection_loss‚2.5515295920833463 1loss/SRqqt2/detection_loss‚1.9960139415480873 0loss/FBt2/assignment_loss‚0.4538521191303933 1loss/SRqqt3/detection_loss‚1.4039871309812253 0loss/FRt3/assignment_loss‚2.7906800482008194 0loss/FBt1/assignment_loss‚0.7761867642402639 /loss/FBt2/detection_loss‚2.7123078869448767 /loss/FRt2/assignment_loss‚4.298706889152527 0loss/FBt4/assignment_loss‚0.3444027481600642 .loss/FBt1/detection_loss‚3.955034111227308 &loss/total_loss‚54.309256290203024 _step‚174 /loss/FRt1/assignment_loss‚4.077421948313713 1loss/SRqqt1/assignment_loss‚2.631789284367715 2loss/SRqqt2/assignment_loss‚2.1317075206474825 1loss/FBt3/assignment_loss‚0.26275082682735496 epoch‚1 trainer/global_step‚2149CŸŒ_\áÖ 0loss/FRt3/assignment_loss‚2.7906800482008194 0loss/FBt1/assignment_loss‚0.7761867642402639 /loss/FBt2/detection_loss‚2.7123078869448767 /loss/FRt2/assignment_loss‚4.298706889152527 0loss/FBt4/assignment_loss‚0.3444027481600642 .loss/FBt1/detection_loss‚3.955034111227308 &loss/total_loss‚54.309256290203024 _step‚174 /loss/FRt1/assignment_loss‚4.077421948313713 1loss/SRqqt1/assignment_loss‚2.631789284367715 2loss/SRqqt2/assignment_loss‚2.1317075206474825 1loss/FBt3/assignment_loss‚0.26275082682735496 epoch‚1 trainer/global_step‚2149 $ _timestamp‚1.788446577329893e+09 _runtime‚1692.801410266 /loss/FRt4/detection_loss‚2.1207254560755904 .loss/FRt2/detection_loss‚2.590565818387109 1loss/SRqqt4/detection_loss‚1.9335164735093713 .loss/FBt3/detection_loss‚1.312135426007444 .loss/FBt4/detection_loss‚2.284528374671936 /loss/FRt4/assignment_loss‚3.945579570824983 2loss/SRqqt3/assignment_loss‚1.7375876899522085 2loss/SRqqt4/assignment_loss‚1.8368736336706206 .loss/FRt1/detection_loss‚4.021948225796223 /loss/FRt3/detection_loss‚2.1394248094823625 1loss/SRqqt1/detection_loss‚2.5515295920833463 1loss/SRqqt2/detection_loss‚1.9960139415480873 0loss/FBt2/assignment_loss‚0.4538521191303933 1loss/SRqqt3/detection_loss‚1.4039871309812253®„§ˆ©“â‚(:ˆ óŽæÔ€Õõü gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚191.909' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.3818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿EM³Jã‚(:¿ óŽæÔÔÒáÿ proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda.out‚ 62.73046875 disk.sda1.out‚0 disk.rbd1.out‚ 8.8671875% disk.nvme0n1.out‚5918.20556640625 disk.dm-1.in‚0 disk.rbd0.in‚0 network.recv‚644154' disk.nvme0c0n1.out‚5918.20556640625 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚5918.20556640625 disk.sda2.in‚0 disk.drbd2265.in‚0) proc.memory.percent‚15.93334674835205 cpu‚25.252632454765827 disk.rbd0.out‚0 network.sent‚ 3.908054e+06$ disk./.usageGB‚298.1416664123535 disk.drbd2080.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0' proc.memory.availableMB‚ 236.8828125 disk.drbd2080.in‚0# memory_percent‚98.5541820526123 disk.dm-1.out‚ 62.73046875 disk.sda3.in‚0 disk.sda.in‚0 disk.drbd1716.in‚0* disk./.usagePercent‚20.008537390453352# proc.memory.rssMB‚ 2610.51953125 disk.sda3.out‚ 62.73046875 disk.rbd1.in‚0²ê†?“ä‚(:ˆ ‚æÔÿ‹ïü gpu.0.gpu‚25.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚183.403' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.6806 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ö&×tIå‚(:¾ ‚æÔ䊇€ disk.rbd1.out‚ 8.9296875 network.sent‚ 4.016676e+06 disk.drbd2265.out‚0 disk.rbd0.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚5966.91259765625 disk.sda3.out‚ 63.7109375 disk.sda2.out‚0 disk.dm-1.in‚0 network.recv‚651330 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 proc.cpu.threads‚17* disk./.usagePercent‚20.005385425691784 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.dm-1.out‚ 63.7109375 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda.out‚ 63.7109375) proc.memory.percent‚15.93334674835205 cpu‚25.25073248502291 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.drbd2266.out‚0" disk.dm-0.out‚5966.91259765625 disk.sda1.in‚0% disk./.usageGB‚298.09469985961914' disk.nvme0c0n1.out‚5966.91259765625$ memory_percent‚98.55420589447021( proc.memory.availableMB‚ 236.87890625# proc.memory.rssMB‚ 2610.51953125XX®Èžæ‚(:“ ‘æÔÍ‘ý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.935' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.186999999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0z½[q8ç‚(:­ ‘æÔþ¬´ÿ disk.drbd2080.out‚0 disk.sda3.in‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.sda1.out‚0# proc.memory.rssMB‚ 2610.51953125 cpu‚25.2488925079921 disk.drbd2266.in‚0 proc.cpu.threads‚17) disk./.usagePercent‚20.00716134995681! disk.nvme0n1.out‚ 6024.8671875 disk.nvme0c0n1.in‚0.078125# disk.nvme0c0n1.out‚ 6024.8671875 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda.out‚ 64.19140625 disk.rbd1.out‚ 8.9296875 disk.drbd1716.out‚0 disk.rbd0.in‚0 disk.dm-0.out‚ 6024.8671875$ memory_percent‚98.55422973632812) proc.memory.percent‚15.93334674835205 network.recv‚654867$ disk./.usageGB‚298.1211624145508 disk.dm-1.out‚ 64.19140625 disk.drbd2265.in‚0 disk.sda3.out‚ 64.19140625 network.sent‚ 4.021841e+06 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.sda1.in‚0# proc.memory.availableMB‚236.875 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0¸µ$†2è- _wandbruntime‚1728  _runtime‚1728D£âÂé¼ $ lr-AdamW/pg1‚0.00012411564505054 $ lr-AdamW/pg2‚0.00012411564505054 trainer/global_step‚2199 $ _timestamp‚1.788446612917437e+09 _runtime‚1728.389070194 _step‚175ŽÞïxÇêÁ $ _timestamp‚1.788446612917437e+09 _runtime‚1728.389070194 _step‚175 $ lr-AdamW/pg1‚0.00012411564505054 $ lr-AdamW/pg2‚0.00012411564505054 trainer/global_step‚2199¯̽Ì2ë- _wandbruntime‚1728  _runtime‚1728(jâyìÓO M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚72746284 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚138038425442816 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚126313871762944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1826800875520 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚139957495703552 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚176 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚12631388976588å¨w`$8 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10415391 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1918870112768 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚124847704810555 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1826144514048 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10415833 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200147968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚2199 _runtime‚1728.390124314 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1826144514048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1919046320640 % _timestamp‚1.7884466129181876e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚72726246 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41202672 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚128140016276992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚128140016276992 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚126313871762944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚139957295555584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28800887 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41222708 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚128140690641408 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚128140690641408 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18385052 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41222708 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28800443 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176207872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚124848376287663 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31523576 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚126313889765888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41222708 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28800443 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176207872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚124848376287663 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31523576 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚126313889765888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚128140016276992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚128140016276992 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚126313871762944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚139957295555584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28800887 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41222708 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚128140690641408 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚128140690641408 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036°€Pv2î- _wandbruntime‚1729  _runtime‚1729Iäsã&ïÝM ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚126370242804736 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18393222 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1820898215315 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1826963941888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚138100100120576 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚138100076180480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚124903425376079 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚2199 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚126370224801792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31537590 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚128197188743680 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚124904096853395 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18393220 _step‚177 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31537590 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41221126 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1826963941888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28813188 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚72758716 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚128197863121408 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10420405 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41221126 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚123083181598720 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 % _timestamp‚1.7884466135942864e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1919896607744 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1919732996096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41241188 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚126370242804736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28813627 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1820243777359 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1827620316672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚140019809176576 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72778780 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚126370224801792 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚72758716 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _runtime‚1729.066451666 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72778780 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31537592 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10420405 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚41221126 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚123083181598720 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 üà ¦^% _timestamp‚1.7884466135942864e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1919896607744 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1919732996096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41241188 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚126370242804736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28813627 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1820243777359 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1827620316672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚140019809176576 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72778780 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚126370224801792 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚72758716 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _runtime‚1729.066451666 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72778780 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31537592 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚128197188743680 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚124904096853395 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18393220 _step‚177 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31537590 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41221126 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1826963941888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28813188 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414± D"2ñ- _wandbruntime‚1729  _runtime‚1729šC—›SòÍ 2loss/SRqqt2/assignment_loss‚1.7431212999792987 /loss/FRt2/detection_loss‚2.6534075297807393 /loss/FBt1/detection_loss‚3.9964508056640624 /loss/FBt3/detection_loss‚1.0741102296803424 1loss/SRqqt4/detection_loss‚1.5391543994729335 trainer/global_step‚2199 0loss/FBt2/assignment_loss‚0.5645919958745935 .loss/FBt2/detection_loss‚2.890423438946406 %loss/total_loss‚57.63834063053485 # _timestamp‚1.78844661359592e+09 _step‚178 0loss/FRt2/assignment_loss‚4.0698474018197315 2loss/SRqqt1/assignment_loss‚2.8887903767247356 /loss/FRt4/detection_loss‚2.3711570919706273 0loss/FRt3/assignment_loss‚2.5297269778592244 1loss/SRqqt3/assignment_loss‚1.749720616291647 0loss/SRqqt1/detection_loss‚2.629860375196703 0loss/SRqqt2/detection_loss‚2.083888053200966 epoch‚1 .loss/FRt4/assignment_loss‚4.14282670387855 0loss/FBt4/assignment_loss‚0.5809717489044286 .loss/FBt4/detection_loss‚3.153985765245226 _runtime‚1729.067409213 2loss/SRqqt4/assignment_loss‚2.0186743600628314 0loss/FBt1/assignment_loss‚0.5738466856250397 0loss/SRqqt3/detection_loss‚1.423394161952685 0loss/FBt3/assignment_loss‚0.2655312617717348 .loss/FRt1/detection_loss‚4.342738111813863 /loss/FRt3/detection_loss‚1.8209812292563063 /loss/FRt1/assignment_loss‚6.5311400095621759³pXóÒ /loss/FRt1/assignment_loss‚6.531140009562175 2loss/SRqqt2/assignment_loss‚1.7431212999792987 /loss/FRt2/detection_loss‚2.6534075297807393 /loss/FBt1/detection_loss‚3.9964508056640624 /loss/FBt3/detection_loss‚1.0741102296803424 1loss/SRqqt4/detection_loss‚1.5391543994729335 trainer/global_step‚2199 0loss/FBt2/assignment_loss‚0.5645919958745935 .loss/FBt2/detection_loss‚2.890423438946406 %loss/total_loss‚57.63834063053485 # _timestamp‚1.78844661359592e+09 _step‚178 0loss/FRt2/assignment_loss‚4.0698474018197315 2loss/SRqqt1/assignment_loss‚2.8887903767247356 /loss/FRt4/detection_loss‚2.3711570919706273 0loss/FRt3/assignment_loss‚2.5297269778592244 1loss/SRqqt3/assignment_loss‚1.749720616291647 0loss/SRqqt1/detection_loss‚2.629860375196703 0loss/SRqqt2/detection_loss‚2.083888053200966 epoch‚1 .loss/FRt4/assignment_loss‚4.14282670387855 0loss/FBt4/assignment_loss‚0.5809717489044286 .loss/FBt4/detection_loss‚3.153985765245226 _runtime‚1729.067409213 2loss/SRqqt4/assignment_loss‚2.0186743600628314 0loss/FBt1/assignment_loss‚0.5738466856250397 0loss/SRqqt3/detection_loss‚1.423394161952685 0loss/FBt3/assignment_loss‚0.2655312617717348 .loss/FRt1/detection_loss‚4.342738111813863 /loss/FRt3/detection_loss‚1.8209812292563063²z˜ñ“ô‚(:ˆ  æÔ­Ž†ý gpu.0.gpu‚49.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.951' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.7902 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0N¨a;õ‚(:°  æÔÔ‡‚ÿ disk.dm-1.in‚0 disk.drbd2265.out‚0$ memory_percent‚98.55461120605469# proc.memory.rssMB‚ 2610.51953125 cpu‚25.24721789981101$ disk./.usageGB‚298.1225814819336 disk.nvme0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda.in‚0 disk.sda2.in‚0$ proc.memory.availableMB‚236.8125$ disk.nvme0c0n1.out‚ 6074.73828125 disk.dm-0.out‚ 6074.73828125 disk.sda2.out‚0 disk.rbd1.in‚0) proc.memory.percent‚15.93334674835205 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda3.out‚ 64.921875 disk.drbd2266.in‚0 disk.sda.out‚ 64.921875" disk.nvme0n1.out‚ 6074.73828125 disk.dm-1.out‚ 64.921875 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 8.9619140625 proc.cpu.threads‚17 network.sent‚ 4.028367e+06 network.recv‚659895 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda1.in‚0* disk./.usagePercent‚20.007256584759546`¹¼ö‚(:’ ¯æÔÔ«“ý gpu.0.gpu‚2.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚125.011' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.002199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ºàO÷‚(:Ä ¯æÔÁäªÿ disk.nvme0n1.in‚0.078125 cpu‚25.245550510732347 disk.drbd1716.in‚0 disk.drbd2266.in‚0$ memory_percent‚98.55465888977051) proc.memory.percent‚15.93334674835205 proc.cpu.threads‚17 disk.sda3.in‚0 disk.sda3.out‚ 65.44140625 network.sent‚ 4.135425e+06 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.out‚ 9.0869140625' proc.memory.availableMB‚ 236.8046875# proc.memory.rssMB‚ 2610.51953125* disk./.usagePercent‚20.008251942052674% disk./.usageGB‚298.13741302490234 disk.sda2.in‚0% disk.nvme0n1.out‚6129.86181640625 disk.dm-1.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.drbd2080.out‚0" disk.dm-0.out‚6129.86181640625 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0 disk.dm-1.out‚ 65.44140625 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚6129.86181640625 disk.rbd0.in‚0 network.recv‚664840 disk.sda.out‚ 65.44140625 disk.drbd1716.out‚0 disk.drbd2265.out‚0ºÙŽ²2ø- _wandbruntime‚1764  _runtime‚1764ÍÁqÈÉùà trainer/global_step‚2249 % _timestamp‚1.7884466495071166e+09 _runtime‚1764.978709405 _step‚179 ' lr-AdamW/pg1‚0.00012385149065022492 ' lr-AdamW/pg2‚0.00012385149065022492Œ6™FÎúÈ ' lr-AdamW/pg2‚0.00012385149065022492 trainer/global_step‚2249 % _timestamp‚1.7884466495071166e+09 _runtime‚1764.978709405 _step‚179 ' lr-AdamW/pg1‚0.00012385149065022492³ήø2û- _wandbruntime‚1764  _runtime‚1764.`ÿeÙ'üÓO QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 _step‚180 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚130998654025216 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚439 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29440056 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1962038626304 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚125773501035520 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚127634418155071 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚129131541708288 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42128724 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚141120962326016 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚129131541708288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10646165 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1867786681344 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1961881292800 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚125773483996160 trainer/global_step‚2249 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚74353098 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚130999328389632 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157333504 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42128724 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181273600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18793454 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32224376 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚129131523705344 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚143082843618816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚127633746677963 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚74373136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32224374 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29439617 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42148760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32224376 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0½{<Õ$ýØO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1867786681344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18793452 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚437 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1860262681803 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 _step‚180 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚130998654025216 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚439 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29440056 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1962038626304 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚125773501035520 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚127634418155071 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚129131541708288 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42128724 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚141120962326016 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚129131541708288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10646165 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1867786681344 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1961881292800 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚125773483996160 trainer/global_step‚2249 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚74353098 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚130999328389632 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157333504 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42128724 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181273600 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18793454 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32224376 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚129131523705344 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚143082843618816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚127633746677963 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚74373136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699017 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32224374 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29439617 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42148760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32224376 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚127689467243487 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74385568 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1867949747712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42147178 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32238390 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32238392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚131056500869632 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚131055826491904 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181260288 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚131056500869632 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29452320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚143145365445120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1861079116767 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32238390 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚181 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42147178 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18801622 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74405632 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚129187876744192 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚453 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10650700 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚125828388126720 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32238392 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42167240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1868606122496 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1861733554723 % _timestamp‚1.7884466501963236e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚129187894747136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29452775 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1962752381440 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74405632 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚141182613063680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚125828405166080 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10651153 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1962909701632 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚143145546705408 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚8447Ùè&€âM BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42167240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1868606122496 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1861733554723 % _timestamp‚1.7884466501963236e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚129187894747136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29452775 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1962752381440 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74405632 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚141182613063680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚125828405166080 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10651153 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1962909701632 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚143145546705408 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚127689467243487 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74385568 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1867949747712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42147178 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32238390 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32238392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚131056500869632 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚131055826491904 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181260288 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚131056500869632 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29452320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚143145365445120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1861079116767 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32238390 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚181 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42147178 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18801622 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74405632 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚129187876744192 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚453 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10650700 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚125828388126720 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32238392µa®›2- _wandbruntime‚1765  _runtime‚1765cy0»T‚Î .loss/FRt2/detection_loss‚2.347826189464993 2loss/SRqqt2/assignment_loss‚2.1508871841689814 2loss/SRqqt3/assignment_loss‚1.7126981924359614 2loss/SRqqt4/assignment_loss‚1.9687448274344206 0loss/FBt1/assignment_loss‚0.7482254902521271 /loss/FBt4/detection_loss‚3.7961975812911986 0loss/FBt4/assignment_loss‚1.1710870504379272 0loss/SRqqt2/detection_loss‚1.975761285294657 1loss/SRqqt4/detection_loss‚1.6571418202947825 /loss/FBt2/detection_loss‚3.0178919159449062 epoch‚1 trainer/global_step‚2249 /loss/FRt1/assignment_loss‚4.811264031073627 /loss/FRt2/assignment_loss‚4.530716058280733 0loss/FRt4/assignment_loss‚3.9358045756816864 1loss/SRqqt1/assignment_loss‚3.242188044216322 /loss/FRt4/detection_loss‚2.1810088495795545 .loss/FRt3/detection_loss‚2.071549677713351 /loss/FBt3/detection_loss‚0.9242833360060029 $ _timestamp‚1.788446650197943e+09 0loss/FBt2/assignment_loss‚0.7770672738547196 /loss/FBt1/detection_loss‚3.7661190750756455 /loss/FRt3/assignment_loss‚2.955007191408764 0loss/SRqqt1/detection_loss‚3.003001119779504 0loss/FBt3/assignment_loss‚0.4380885212159046 .loss/FRt1/detection_loss‚2.918406465474297 0loss/SRqqt3/detection_loss‚1.427748858011686 %loss/total_loss‚57.52871461439175 _step‚182 _runtime‚1765.669419378Y¾ñYƒÓ .loss/FRt2/detection_loss‚2.347826189464993 2loss/SRqqt2/assignment_loss‚2.1508871841689814 2loss/SRqqt3/assignment_loss‚1.7126981924359614 2loss/SRqqt4/assignment_loss‚1.9687448274344206 0loss/FBt1/assignment_loss‚0.7482254902521271 /loss/FBt4/detection_loss‚3.7961975812911986 0loss/FBt4/assignment_loss‚1.1710870504379272 0loss/SRqqt2/detection_loss‚1.975761285294657 1loss/SRqqt4/detection_loss‚1.6571418202947825 /loss/FBt2/detection_loss‚3.0178919159449062 epoch‚1 trainer/global_step‚2249 /loss/FRt1/assignment_loss‚4.811264031073627 /loss/FRt2/assignment_loss‚4.530716058280733 0loss/FRt4/assignment_loss‚3.9358045756816864 1loss/SRqqt1/assignment_loss‚3.242188044216322 /loss/FRt4/detection_loss‚2.1810088495795545 .loss/FRt3/detection_loss‚2.071549677713351 /loss/FBt3/detection_loss‚0.9242833360060029 $ _timestamp‚1.788446650197943e+09 0loss/FBt2/assignment_loss‚0.7770672738547196 /loss/FBt1/detection_loss‚3.7661190750756455 /loss/FRt3/assignment_loss‚2.955007191408764 0loss/SRqqt1/detection_loss‚3.003001119779504 0loss/FBt3/assignment_loss‚0.4380885212159046 .loss/FRt1/detection_loss‚2.918406465474297 0loss/SRqqt3/detection_loss‚1.427748858011686 %loss/total_loss‚57.52871461439175 _step‚182 _runtime‚1765.669419378¶O\ãjž„‚(:“ ¾æÔǧ™ý gpu.0.gpu‚43.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚143.802' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.760399999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0È'4G…‚(:¼ ¾æÔØî›ÿ# disk./.usageGB‚298.151180267334 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.drbd1716.out‚0# proc.memory.rssMB‚ 2610.51953125 network.sent‚ 4.142067e+06 disk.drbd2265.out‚0( proc.memory.availableMB‚ 236.37890625 disk.rbd1.in‚0 network.recv‚670185 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0) proc.memory.percent‚15.93334674835205 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.sda.out‚ 66.140625 cpu‚25.243923526830887 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.in‚0 disk.drbd1716.in‚0% disk.nvme0n1.out‚6178.58056640625$ memory_percent‚98.55725765228271) disk./.usagePercent‚20.00917587324375 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚6178.58056640625" disk.dm-0.out‚6178.58056640625 disk.sda.in‚0 disk.dm-1.out‚ 66.140625 disk.rbd1.out‚ 9.1494140625 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 66.140625 disk.rbd0.in‚0 disk.sda1.out‚0 proc.cpu.threads‚1794¾Q“†‚(:ˆ ÍæÔ┃ý gpu.0.gpu‚39.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.013' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0026 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÒÌ÷ßC‡‚(:¸ ÍæÔ“´Žÿ network.recv‚674973% disk./.usageGB‚298.18152236938477 disk.sda3.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd1716.out‚0 disk.sda3.out‚ 66.7265625 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.drbd1716.in‚0# proc.memory.availableMB‚236.375# proc.memory.rssMB‚ 2610.51953125 disk.rbd0.in‚0 disk.rbd1.in‚0$ disk.nvme0n1.out‚6245.1923828125 disk.drbd2265.in‚0 disk.drbd2080.in‚0 cpu‚25.242323313341018 network.sent‚ 4.2494e+06 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0& disk.nvme0c0n1.out‚6245.1923828125* disk./.usagePercent‚20.011212157171094 disk.sda.out‚ 66.7265625 disk.sda2.out‚0 disk.drbd2265.out‚0 proc.cpu.threads‚17 disk.dm-0.in‚0.078125 disk.sda1.out‚0) proc.memory.percent‚15.93334674835205! disk.dm-0.out‚6245.1923828125 disk.rbd1.out‚ 9.1494140625 disk.dm-1.out‚ 66.7265625 disk.drbd2080.out‚0$ memory_percent‚98.55728149414062½¹ü™“ˆ‚(:ˆ ÜæÔˆÓóü gpu.0.gpu‚41.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚128.809' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.7618 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0dž I‰‚(:¾ ÜæÔ²ÿüÿ% disk./.usageGB‚298.18976974487305 disk.dm-0.in‚0.078125 disk.sda1.in‚0# proc.memory.rssMB‚ 2610.51953125) proc.memory.percent‚15.93334674835205$ disk.nvme0n1.out‚6294.7822265625 disk.drbd1716.in‚0! disk.dm-0.out‚6294.7822265625 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚98.55737686157227 network.sent‚ 4.255798e+06 disk.dm-1.in‚0 disk.sda3.out‚ 67.26953125 cpu‚25.24085834469177 disk.dm-1.out‚ 67.26953125 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0* disk./.usagePercent‚20.011765645460123 disk.rbd1.in‚0 disk.rbd1.out‚ 9.244140625 disk.rbd0.out‚0 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda.out‚ 67.26953125 disk.sda2.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0& disk.nvme0c0n1.out‚6294.7822265625 disk.drbd2266.out‚0& proc.memory.availableMB‚ 236.359375 network.recv‚679811vfÄÍ2Š- _wandbruntime‚1801  _runtime‚1801êPñ¢Æ‹À & lr-AdamW/pg1‚0.0001235747263391359 & lr-AdamW/pg2‚0.0001235747263391359 trainer/global_step‚2299 $ _timestamp‚1.788446686299734e+09 _runtime‚1801.771422521 _step‚183¶‘9ˌŠtrainer/global_step‚2299 $ _timestamp‚1.788446686299734e+09 _runtime‚1801.771422521 _step‚183 & lr-AdamW/pg1‚0.0001235747263391359 & lr-AdamW/pg2‚0.0001235747263391359·§®×2- _wandbruntime‚1801  _runtime‚1801ÀÇ’ŽÓO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32925174 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1901752458959 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30076801 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19201854 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10874947 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚146208341956608 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1908772487168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191759360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚128518690524160 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚75979950 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32925176 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚133857291773440 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1908772487168 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚75999988 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚131949193650688 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19201852 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚184 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43054776 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚133857291773440 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚144203499209216 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚130419788545371 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43074812 H>DeviceStatsMonitor.on_train_batch_start/ac‰À †Ètive.small_pool.peak‚24126 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚146208533715968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚2299 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚144203523149312 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43054776 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30076361 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1901098021211 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0ùÅÈÞ'ØO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚133857966137856 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚131949175647744 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚133857966137856 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167819264 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚128518707563520 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43074812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚146208533715968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚2299 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚144203523149312 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43054776 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚30076361 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1901098021211 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32925174 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1901752458959 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30076801 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19201854 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10874947 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚146208341956608 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1908772487168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191759360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚128518690524160 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚75979950 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32925176 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚133857291773440 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1908772487168 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚75999988 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚131949193650688 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19201852 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚184 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43054776 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚133857291773440 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚144203499209216 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚130419788545371 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43074812 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884466869992886e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43073230 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚457 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚130475509110895 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚144265149946880 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19210020 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚133915138617856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚132005528686592 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165708800 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30089504 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30089045 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚144265173886976 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚128573594654720 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚133914464240128 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚132005528686592 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1908935553536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚130476180588211 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚128573611694080 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚132005546689536 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32939192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43093292 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1908935553536 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32939190 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76032484 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚459 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19210022 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1901914456175 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1909591928320 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚146270850155520 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2005700208640 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 _runtime‚1802.471399261 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚132005546689536 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189648896 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30089504 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30089045 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚144265173886976 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚128573594654720 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚133914464240128 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚132005528686592 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1908935553536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚130476180588211 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚128573611694080 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚132005546689536 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32939192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43093292 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24399 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1908935553536 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32939190 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76032484 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚459 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19210022 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1901914456175 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 C9DeviceSt…ò‰„Æ atsMonitor.on_train_batch_end/allocation.all.peak‚24399 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1909591928320 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚146270850155520 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2005700208640 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 _runtime‚1802.471399261 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚132005546689536 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189648896 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884466869992886e+09¹vädM2“- _wandbruntime‚1802  _runtime‚1802Ì *±V”Ð % _timestamp‚1.7884466870008724e+09 0loss/FBt1/assignment_loss‚0.4680640228530869 /loss/SRqqt1/detection_loss‚2.89767564194543 epoch‚1 /loss/FBt3/detection_loss‚1.0161370334714868 trainer/global_step‚2299 /loss/FBt4/detection_loss‚2.6586444019339988 /loss/FRt1/assignment_loss‚5.725728799899419 0loss/SRqqt1/assignment_loss‚3.06860460128103 2loss/SRqqt2/assignment_loss‚1.9553613510727883 /loss/FRt4/detection_loss‚2.0496959586938224 1loss/SRqqt4/detection_loss‚1.7829975964593106 2loss/SRqqt3/assignment_loss‚1.5363197704960763 2loss/SRqqt4/assignment_loss‚2.1100709912229756 1loss/SRqqt3/detection_loss‚1.3626023718798663 _runtime‚1802.472406542 /loss/FRt2/assignment_loss‚3.930915137132009 .loss/FRt2/detection_loss‚2.300004089872042 _step‚186 1loss/SRqqt2/detection_loss‚1.7978427249193192 /loss/FBt1/detection_loss‚2.8792773415060604 /loss/FBt2/detection_loss‚3.2670068277253046 %loss/total_loss‚54.18198845924219 /loss/FRt4/assignment_loss‚3.305323638297893 0loss/FBt4/assignment_loss‚0.6072141055483371 .loss/FRt1/detection_loss‚3.244143471121788 0loss/FRt3/assignment_loss‚3.0933647269294378 0loss/FBt2/assignment_loss‚0.5903928213649325 1loss/FBt3/assignment_loss‚0.20336279538613863 .loss/FRt3/detection_loss‚2.331238238229638gk©[•Õ /loss/FRt2/assignment_loss‚3.930915137132009 .loss/FRt2/detection_loss‚2.300004089872042 _step‚186 1loss/SRqqt2/detection_loss‚1.7978427249193192 /loss/FBt1/detection_loss‚2.8792773415060604 /loss/FBt2/detection_loss‚3.2670068277253046 %loss/total_loss‚54.18198845924219 /loss/FRt4/assignment_loss‚3.305323638297893 0loss/FBt4/assignment_loss‚0.6072141055483371 .loss/FRt1/detection_loss‚3.244143471121788 0loss/FRt3/assignment_loss‚3.0933647269294378 0loss/FBt2/assignment_loss‚0.5903928213649325 1loss/FBt3/assignment_loss‚0.20336279538613863 .loss/FRt3/detection_loss‚2.331238238229638 % _timestamp‚1.7884466870008724e+09 0loss/FBt1/assignment_loss‚0.4680640228530869 /loss/SRqqt1/detection_loss‚2.89767564194543 epoch‚1 /loss/FBt3/detection_loss‚1.0161370334714868 trainer/global_step‚2299 /loss/FBt4/detection_loss‚2.6586444019339988 /loss/FRt1/assignment_loss‚5.725728799899419 0loss/SRqqt1/assignment_loss‚3.06860460128103 2loss/SRqqt2/assignment_loss‚1.9553613510727883 /loss/FRt4/detection_loss‚2.0496959586938224 1loss/SRqqt4/detection_loss‚1.7829975964593106 2loss/SRqqt3/assignment_loss‚1.5363197704960763 2loss/SRqqt4/assignment_loss‚2.1100709912229756 1loss/SRqqt3/detection_loss‚1.3626023718798663 _runtime‚1802.472406542º‘”¶G“–‚(:ˆ ëæÔËÈáý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.503' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.9006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0—ÚbÀD—‚(:¹ ëæÔ…߸€ disk.sda2.in‚0# disk.nvme0n1.out‚6378.478515625 disk.drbd1716.in‚0% disk.nvme0c0n1.out‚6378.478515625 disk.drbd2266.in‚0 disk.sda.out‚ 67.8984375 disk.drbd2080.out‚0 disk.sda3.in‚0 network.recv‚683152 cpu‚25.239292344000837 proc.cpu.threads‚17* disk./.usagePercent‚20.012195738117647 disk.rbd1.out‚ 9.455078125 disk.dm-1.out‚ 67.8984375 disk.drbd2265.in‚0 disk.dm-0.out‚6378.478515625# proc.memory.rssMB‚ 2610.51953125$ disk./.usageGB‚298.1961784362793 disk.sda3.out‚ 67.8984375 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0' proc.memory.availableMB‚ 236.3046875) proc.memory.percent‚15.93334674835205 network.sent‚ 4.260892e+06 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0$ memory_percent‚98.55771064758301$Dg“˜‚(:ˆ úæÔ­¢„ý gpu.0.gpu‚37.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚127.052' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.4104 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“gG™‚(:¼ úæÔÂÂŒ€ disk.sda2.out‚0 disk.dm-1.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚6431.275390625 disk.rbd0.out‚0 proc.cpu.threads‚17 network.recv‚690105 disk.drbd1716.in‚0 disk.sda3.in‚0 disk.nvme0n1.in‚0.078125# disk.nvme0n1.out‚6431.275390625$ memory_percent‚98.55775833129883# proc.memory.rssMB‚ 2610.51953125 disk.drbd1716.out‚0 disk.drbd2266.out‚0% disk./.usageGB‚298.20689392089844 disk.drbd2080.out‚0 disk.sda3.out‚ 69.26171875 disk.sda1.out‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.out‚6431.275390625 disk.sda.in‚0 disk.rbd1.in‚0 cpu‚25.237775354121272 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0& proc.memory.availableMB‚ 236.296875* disk./.usagePercent‚20.012914863281328 disk.sda.out‚ 69.26171875 disk.rbd1.out‚ 9.603515625) proc.memory.percent‚15.93334674835205 network.sent‚ 4.369633e+06 disk.dm-1.out‚ 69.26171875 disk.drbd2080.in‚0{>(„2š- _wandbruntime‚1838  _runtime‚1838}´,¤Ç›Á _runtime‚1838.386534417 _step‚187 & lr-AdamW/pg1‚0.0001232854097945406 & lr-AdamW/pg2‚0.0001232854097945406 trainer/global_step‚2349 % _timestamp‚1.7884467229149427e+09EÎRÞÌœÆ _step‚187 & lr-AdamW/pg1‚0.0001232854097945406 & lr-AdamW/pg2‚0.0001232854097945406 trainer/global_step‚2349 % _timestamp‚1.7884467229149427e+09 _runtime‚1838.386534417»€ÿBž2- _wandbruntime‚1838  _runtime‚1838” ^!Ù'žÓO @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚136716603886080 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361472 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1941933360619 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚134766827590144 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _step‚188 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚134766845593088 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1949758292992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚133205830412779 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚149334031591936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚136715929521664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 174110720 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43980828 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚131263914091520 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33625974 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚149333833541120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44000864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11104175 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2047971559424 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33625976 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚136715929521664 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 % _timestamp‚1.7884467229156163e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1949758292992 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚134766827590144 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚134766845593088 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 198050816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚136715929521664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863814656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 174110720 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437748 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43980828 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚131263914091520 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33625974 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚149333833541120 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44000864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11104175 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2047971559424 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33625976 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚136715929521664 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 % _timestamp‚1.7884467229156163e+09 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1949758292992 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚134766827590144 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚134766845593088 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 198050816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1949758292992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚133205830412779 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361472 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚149334031591936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1950577734144 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551744 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2048657634816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚134823180628992 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.curreÝ>9ïŒnt‚18002944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚131318818222080 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33639990 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77639272 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30727574 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚1839.072641785 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44019344 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44019344 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2048821246464 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43999282 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚149396344464896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43999282 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1949921359360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33639992 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30727125 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11109152 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _step‚189 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚134823198631936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19618422 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚131318801182720 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 $ _timestamp‚1.788446723600561e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚149396532016640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚2349 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚134823180628992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1943404233539 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77659336 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚133261550978303 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚136773776366080’·BÈç&¢áM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1950577734144 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551744 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2048657634816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚134823180628992 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477316 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚131318818222080 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33639990 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77639272 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30727574 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚1839.072641785 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44019344 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44019344 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2048821246464 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43999282 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚149396344464896 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43999282 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1949921359360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33639992 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30727125 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11109152 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _step‚189 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚134823198631936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19618422 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚131318801182720 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 $ _timestamp‚1.788446723600561e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚149396532016640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚2349 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚134823180628992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1943404233539 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77659336 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚133261550978303 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚136773776366080 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11108705 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017½`FÊ2£- _wandbruntime‚1839  _runtime‚1839õû1½U¤Ï /loss/FRt1/detection_loss‚2.9201386705521615 epoch‚1 $ _timestamp‚1.788446723602187e+09 2loss/SRqqt1/assignment_loss‚2.9195793350537618 /loss/FRt4/detection_loss‚2.0871218823995745 0loss/SRqqt2/detection_loss‚2.243004759383756 /loss/SRqqt3/detection_loss‚1.50815704533687 trainer/global_step‚2349 /loss/FRt4/assignment_loss‚3.612639759407669 /loss/FRt2/detection_loss‚2.2055833197158314 0loss/SRqqt1/detection_loss‚2.547353877623876 .loss/FBt1/detection_loss‚4.631352805429035 /loss/FBt2/detection_loss‚2.5383832938969135 -loss/FBt4/detection_loss‚4.66850033071306 _runtime‚1839.073740259 _step‚190 /loss/FRt2/assignment_loss‚4.463492466055828 &loss/total_loss‚55.480160027660304 2loss/SRqqt4/assignment_loss‚1.8378051524279548 1loss/SRqqt4/detection_loss‚1.6603529353122242 /loss/FBt3/detection_loss‚0.8549204052975162 0loss/FRt3/assignment_loss‚1.8992458317014906 2loss/SRqqt2/assignment_loss‚2.1268679553339647 1loss/FBt1/assignment_loss‚0.44970801224289086 0loss/FBt4/assignment_loss‚0.7001805550987936 -loss/FRt3/detection_loss‚2.15831107687619 2loss/SRqqt3/assignment_loss‚1.4782875566528393 1loss/FBt3/assignment_loss‚0.42543115650266145 /loss/FRt1/assignment_loss‚4.844795496233048 0loss/FBt2/assignment_loss‚0.69894634841239174ÉùtZ¥Ô 2loss/SRqqt4/assignment_loss‚1.8378051524279548 1loss/SRqqt4/detection_loss‚1.6603529353122242 /loss/FBt3/detection_loss‚0.8549204052975162 0loss/FRt3/assignment_loss‚1.8992458317014906 2loss/SRqqt2/assignment_loss‚2.1268679553339647 1loss/FBt1/assignment_loss‚0.44970801224289086 0loss/FBt4/assignment_loss‚0.7001805550987936 -loss/FRt3/detection_loss‚2.15831107687619 2loss/SRqqt3/assignment_loss‚1.4782875566528393 1loss/FBt3/assignment_loss‚0.42543115650266145 /loss/FRt1/assignment_loss‚4.844795496233048 0loss/FBt2/assignment_loss‚0.6989463484123917 /loss/FRt1/detection_loss‚2.9201386705521615 epoch‚1 $ _timestamp‚1.788446723602187e+09 2loss/SRqqt1/assignment_loss‚2.9195793350537618 /loss/FRt4/detection_loss‚2.0871218823995745 0loss/SRqqt2/detection_loss‚2.243004759383756 /loss/SRqqt3/detection_loss‚1.50815704533687 trainer/global_step‚2349 /loss/FRt4/assignment_loss‚3.612639759407669 /loss/FRt2/detection_loss‚2.2055833197158314 0loss/SRqqt1/detection_loss‚2.547353877623876 .loss/FBt1/detection_loss‚4.631352805429035 /loss/FBt2/detection_loss‚2.5383832938969135 -loss/FBt4/detection_loss‚4.66850033071306 _runtime‚1839.073740259 _step‚190 /loss/FRt2/assignment_loss‚4.463492466055828 &loss/total_loss‚55.480160027660304¾¹ôG“¦‚(:ˆ ‰æÔº»Žý gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚189.262' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.8524 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0j†D¨G§‚(:¼ ‰æÔ§¦ªÿ) disk./.usagePercent‚20.01331346701214 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 69.55078125 disk.rbd1.out‚ 9.603515625) proc.memory.percent‚15.93334674835205 network.recv‚693246 disk.sda3.out‚ 69.55078125 disk.sda2.in‚0 disk.dm-1.out‚ 69.55078125 disk.drbd2265.in‚0# memory_percent‚98.5579252243042 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.drbd2080.in‚0& disk.nvme0c0n1.out‚6493.0458984375! disk.dm-0.out‚6493.0458984375 disk.drbd2265.out‚0 cpu‚25.23616026737913# disk./.usageGB‚298.212833404541 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.rbd0.out‚0 disk.sda1.in‚0$ disk.nvme0n1.out‚6493.0458984375 disk.drbd2266.in‚0 disk.sda.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125# proc.memory.rssMB‚ 2610.51953125 proc.cpu.threads‚17 network.sent‚ 4.374828e+06 disk.dm-0.in‚0.078125 disk.dm-1.in‚0( proc.memory.availableMB‚ 236.26953125kˆ»’¨‚(:‡ ˜æÔÑÎŽý gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚192.513' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.5026 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0d³x O©‚(:Ä ˜æÔ¥Œµÿ# proc.memory.rssMB‚ 2610.51953125' disk.nvme0c0n1.out‚6542.66943359375$ memory_percent‚98.55954647064209 proc.cpu.threads‚17% disk./.usageGB‚298.22900772094727 disk.sda2.out‚0 disk.rbd1.out‚ 9.9013671875% disk.nvme0n1.out‚6542.66943359375 network.sent‚ 4.483352e+06 disk.drbd2080.in‚0 disk.sda3.out‚ 70.19921875 disk.drbd2266.out‚0 disk.rbd0.in‚0" disk.dm-0.out‚6542.66943359375 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.sda.in‚0( proc.memory.availableMB‚ 236.00390625 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 70.19921875 disk.drbd1716.out‚0) proc.memory.percent‚15.93334674835205 cpu‚25.23467986598965 disk.dm-1.out‚ 70.19921875 disk.drbd2080.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 network.recv‚700309* disk./.usagePercent‚20.014398938957317 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0gœæ“ª‚(:ˆ §æÔè…ý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.523' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.3046 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¥û÷šP«‚(:Å §æÔ›•ã€ network.recv‚703701 disk.rbd0.out‚0 network.sent‚ 4.488491e+06* disk./.usagePercent‚20.014636513949092 disk.sda1.out‚0 disk.dm-1.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0( proc.memory.availableMB‚ 236.01171875% disk./.usageGB‚298.23254776000977 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚6604.29345703125 disk.drbd2266.out‚0 disk.rbd0.in‚0# proc.memory.rssMB‚ 2610.51953125 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0 disk.sda3.out‚ 70.71484375 disk.dm-0.in‚0.078125" disk.dm-0.out‚6604.29345703125 disk.sda.out‚ 70.71484375$ memory_percent‚98.55949878692627 disk.sda2.out‚0 disk.drbd2265.out‚0) proc.memory.percent‚15.93334674835205 proc.cpu.threads‚17 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.sda.in‚0 cpu‚25.23318725477905 disk.sda1.in‚0 disk.rbd1.out‚ 10.0263671875 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.dm-1.out‚ 70.71484375% disk.nvme0n1.out‚6604.29345703125 disk.drbd2265.in‚0ú ÞÝ2¬- _wandbruntime‚1875  _runtime‚1875Æ@kÈ­Â ' lr-AdamW/pg1‚0.00012298360130957324 ' lr-AdamW/pg2‚0.00012298360130957324 trainer/global_step‚2399 $ _timestamp‚1.788446759628405e+09 _runtime‚1875.100000895 _step‚191S`Õiꨂ ' lr-AdamW/pg1‚0.00012298360130957324 ' lr-AdamW/pg2‚0.00012298360130957324 trainer/global_step‚2399 $ _timestamp‚1.788446759628405e+09 _runtime‚1875.100000895 _step‚191¿Ž~G—2¯- _wandbruntime‚1875  _runtime‚1875PÀô“¥°ÒO ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34326774 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚135992543757295 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚150368596915712 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚137584497535488 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31353286 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚137584479532544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1990744098816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34326774 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚79233654 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚139575241634304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44926916 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477108 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20038 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11334632 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34326776 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34326776 _step‚192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863814656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189662208 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2091122470912 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2090956748800 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1983423137775 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚139574567269888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚152459529724416 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚152459719386624 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚137584479532544 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1990087737344 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11334179 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚139574567269888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364416 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364416 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚152459529724416 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚152459719386624 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚137584479532544 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1990087737344 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11334179 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24126 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34326774 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24399 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚135992543757295 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24399 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚150368596915712 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853513 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20036 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚137584497535488 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31353286 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20036 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚137584479532544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1990744098816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 250950656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34326774 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24126 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚79233654 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚139575241634304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944ÀJG2²- _wandbruntime‚1876  _runtime‚1876\ÿoâ&³ÜM SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656374784 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 250950656 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34340790 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2091981632000 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44925334 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11338803 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11339255 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884467605600939e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863814656 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚137640832571392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853513 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34340792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1984239572947 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34340792 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31366077 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚150430223713280 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚139632414114304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2091818020352 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1990907165184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚137640850574336 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79286188 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31365623 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1991563539968 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚137640832571392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _step‚193 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚139632414114304 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚152522041733632 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1983585134991 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79286188 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79266124 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚152522229285376 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551744 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44945396 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚134064024750080 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚137640850574336 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44925334 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚ 1876.03224783 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚79266124 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674377728 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674377728 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34340792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1984239572947 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34340792 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31366077 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚150430223713280 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚139632414114304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2091818020352 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1990907165184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚137640850574336 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79286188 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31365623 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437956 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1991563539968 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20062 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚137640832571392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24399 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24126 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _step‚193 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚139632414114304 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚152522041733632 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1983585134991 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79286188 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79266124 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚152522229285376 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551744 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚44945396 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863814656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚134064024750080 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585024 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚137640850574336 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44925334 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚ 1876.03224783 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚79266124 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34340790 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2091981632000 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699017 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44925334 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11338803 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20062 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11339255 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884467605600939e+09 SEDeviceStatsMonitor.on_train_batcE»ëÌ.h_end/allocated_bytes.small_pool.peak‚ 863814656 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚137640832571392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656374784Áú‹~]2µ- _wandbruntime‚1876  _runtime‚1876&¹© X¶Ò 0loss/FRt3/assignment_loss‚2.7137434685772117 1loss/FBt3/assignment_loss‚0.19115866465018225 0loss/SRqqt2/detection_loss‚2.056186872270872 .loss/FBt1/detection_loss‚4.172800050841437 %loss/total_loss‚54.33146894786546 1loss/SRqqt1/detection_loss‚3.0273121916330776 /loss/FRt2/assignment_loss‚4.365018969927078 2loss/SRqqt2/assignment_loss‚2.0400163973769487 1loss/FBt2/assignment_loss‚0.21692787849569867 1loss/FBt4/assignment_loss‚0.20091600418709277 _step‚194 1loss/SRqqt3/detection_loss‚1.8363413863948412 epoch‚1 /loss/FRt4/assignment_loss‚3.677721998584804 1loss/SRqqt4/detection_loss‚1.6104451777443054 1loss/SRqqt1/assignment_loss‚2.130937325266691 /loss/FBt3/detection_loss‚1.6379021497197308 trainer/global_step‚2399 2loss/SRqqt4/assignment_loss‚1.7520802179499277 0loss/FBt1/assignment_loss‚0.4487898213148027 .loss/FRt3/detection_loss‚2.184480686079372 .loss/FRt4/detection_loss‚2.071861360500108 /loss/FRt1/assignment_loss‚4.490844090779622 /loss/FRt1/detection_loss‚2.6822593334393625 .loss/FRt2/detection_loss‚2.560802027201041 .loss/FBt2/detection_loss‚2.879920315742493 _runtime‚1876.033337677 2loss/SRqqt3/assignment_loss‚1.5237514176405966 /loss/FBt4/detection_loss‚3.8592511415481567 % _timestamp‚1.7884467605618005e+09@z á]·× 1loss/SRqqt1/assignment_loss‚2.130937325266691 /loss/FBt3/detection_loss‚1.6379021497197308 trainer/global_step‚2399 2loss/SRqqt4/assignment_loss‚1.7520802179499277 0loss/FBt1/assignment_loss‚0.4487898213148027 .loss/FRt3/detection_loss‚2.184480686079372 .loss/FRt4/detection_loss‚2.071861360500108 /loss/FRt1/assignment_loss‚4.490844090779622 /loss/FRt1/detection_loss‚2.6822593334393625 .loss/FRt2/detection_loss‚2.560802027201041 .loss/FBt2/detection_loss‚2.879920315742493 _runtime‚1876.033337677 2loss/SRqqt3/assignment_loss‚1.5237514176405966 /loss/FBt4/detection_loss‚3.8592511415481567 % _timestamp‚1.7884467605618005e+09 0loss/FRt3/assignment_loss‚2.7137434685772117 1loss/FBt3/assignment_loss‚0.19115866465018225 0loss/SRqqt2/detection_loss‚2.056186872270872 .loss/FBt1/detection_loss‚4.172800050841437 %loss/total_loss‚54.33146894786546 1loss/SRqqt1/detection_loss‚3.0273121916330776 /loss/FRt2/assignment_loss‚4.365018969927078 2loss/SRqqt2/assignment_loss‚2.0400163973769487 1loss/FBt2/assignment_loss‚0.21692787849569867 1loss/FBt4/assignment_loss‚0.20091600418709277 _step‚194 1loss/SRqqt3/detection_loss‚1.8363413863948412 epoch‚1 /loss/FRt4/assignment_loss‚3.677721998584804 1loss/SRqqt4/detection_loss‚1.6104451777443054 ð‡-“¸‚(:ˆ ¶æÔ£«œý gpu.0.gpu‚48.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚123.941' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.7882 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0œ ŽF¹‚(:» ¶æÔ¯€¾ÿ disk.rbd0.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0 disk.sda1.in‚0 network.sent‚ 4.495085e+06 network.recv‚708695 disk.drbd2080.out‚0" disk.dm-0.out‚6648.54345703125 disk.sda.out‚ 71.3828125 proc.memory.rssMB‚ 2635.1875$ disk./.usageGB‚298.2378273010254 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0$ memory_percent‚98.70929718017578 cpu‚25.234396371511725' disk.nvme0c0n1.out‚6648.54345703125 disk.sda3.out‚ 71.3828125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.drbd2265.out‚0* proc.memory.percent‚16.083908081054688) disk./.usagePercent‚20.01499082837648 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.rbd1.out‚ 10.0888671875 disk.dm-1.out‚ 71.3828125 disk.sda2.out‚0% disk.nvme0n1.out‚6648.54345703125% proc.memory.availableMB‚ 211.46875 disk.sda3.in‚0 disk.drbd1716.in‚0ù'!º‚(:’ ÅæÔö©žý gpu.0.gpu‚9.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚123.266' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.653200000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0NÅ¥ŽI»‚(:¾ ÅæÔõÓ§ÿ% disk./.usageGB‚298.23331451416016 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚6685.57080078125 disk.sda3.out‚ 71.8515625 disk.sda.in‚0$ memory_percent‚98.71034622192383 network.recv‚713767 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 proc.memory.rssMB‚ 2635.1875 disk.dm-1.in‚0 disk.sda.out‚ 71.8515625 disk.rbd1.out‚ 10.0888671875 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda1.out‚0& proc.memory.availableMB‚ 211.296875 cpu‚25.271100283915644 disk.sda2.in‚0 network.sent‚ 4.602434e+06 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚6685.57080078125* proc.memory.percent‚16.083908081054688 disk.dm-1.out‚ 71.8515625 disk.drbd1716.out‚0 disk.drbd2080.out‚0* disk./.usagePercent‚20.014687971463474% disk.nvme0n1.out‚6685.57080078125 disk.drbd2265.in‚0 disk.rbd0.out‚0 proc.cpu.threads‚17¢­Æ|¼‚(:’ ÔæÔ“ÿþü gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.969' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.593799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0?á–®G½‚(:¼ ÔæÔé‚šÿ disk.sda3.out‚ 72.4453125 disk.drbd2266.in‚0$ disk./.usageGB‚298.2399024963379 disk.drbd2080.out‚0* proc.memory.percent‚16.083908081054688 network.recv‚718732 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 72.4453125 disk.drbd1716.in‚0 disk.drbd2265.in‚0* disk./.usagePercent‚20.015130096475108 disk.sda.in‚0' disk.nvme0c0n1.out‚6735.15673828125 disk.drbd2266.out‚0 disk.sda.out‚ 72.4453125( proc.memory.availableMB‚ 211.71484375" disk.dm-0.out‚6735.15673828125 disk.rbd1.in‚0 disk.rbd1.out‚ 10.12109375% disk.nvme0n1.out‚6735.15673828125 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 proc.cpu.threads‚17 network.sent‚ 4.608858e+06 disk.sda2.out‚0 disk.dm-1.in‚0$ memory_percent‚98.70779514312744 disk.drbd2265.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda1.out‚0 cpu‚25.30621140173047 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 proc.memory.rssMB‚ 2635.1875²äݾ‚(:’ ãæÔýí„þ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.827' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.565399999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¤º8ØE¿‚(:º ãæÔ¼¨ÿ% disk.nvme0n1.out‚6790.23779296875 disk.dm-1.in‚0 disk.dm-1.out‚ 72.8671875 disk.sda3.out‚ 72.8671875" disk.dm-0.out‚6790.23779296875 disk.sda1.out‚0 disk.drbd2265.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.sda1.in‚0 disk.drbd2080.out‚0 disk.sda2.out‚0* proc.memory.percent‚16.083908081054688 network.sent‚ 4.613863e+06 disk.rbd1.in‚0' disk.nvme0c0n1.out‚6790.23779296875 disk.sda.out‚ 72.8671875 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0' proc.memory.availableMB‚ 211.0390625 proc.memory.rssMB‚ 2635.1875 disk.rbd0.out‚0 disk.drbd2265.in‚0) disk./.usagePercent‚20.01541580088332 disk.rbd1.out‚ 10.24609375 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0$ disk./.usageGB‚298.2441596984863 disk.drbd1716.in‚0# memory_percent‚98.7119197845459 cpu‚25.341416770020082 proc.cpu.threads‚17 network.recv‚721650Ï2À- _wandbruntime‚1937  _runtime‚1937¾€VM@*ÁºT ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚20492542 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15975 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24399 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚138798464034935 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚136781693315072 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚20492540 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914384 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 trainer/global_step‚2417 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚45619235 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 2023728806912 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863814656 YLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚35122574 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚11642226 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚155521460392960 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9955585024 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚35122572 LCDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak‚2605 \JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚ 2023728806912 LDDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.peak‚908 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953744 NADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚80757784 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863814656 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 NADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚80741807 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853513 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492829184 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚138797956081191 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510832128 OIDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.current‚0 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492829184 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 FDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955585024 YLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚45635210 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 VIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚32134768 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚155521323205120 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚140459858681856 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15977 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚35122572 UHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚45619235 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 113247744 REDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚80757784 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 UHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚45635210 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚140459858681856 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 2024221636096 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚11642078 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 TEDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.peak‚ 9643699017 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 2016770719863 _step‚195 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15975 [GDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated‚142484098320896 bNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚140459876684800 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510832128 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚142483587488768 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚153395847932928 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 _runtime‚1937.950415345 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚142483587488768 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚148 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 2016279805479 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚35122574 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24126 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 2024221636096 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚142484098320896 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚140459876684800 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 2125588519936 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚150 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15977 REDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚32134618 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 _runtime‚1937.950415345 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚142483587488768 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚148 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 2016279805479 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚35122574 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24126 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 2024221636096 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚142484098320896 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚140459876684800 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 2125588519936 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚150 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15977 REDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚32134618 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955585024 YLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚45635210 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 VIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚32134768 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚155521323205120 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚140459858681856 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15977 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚35122572 UHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚45619235 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 113247744 REDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚80757784 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 UHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚45635210 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚140459858681856 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 2024221636096 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚11642078 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 TEDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.peak‚ 9643699017ÃËÕ®+2Ã- _wandbruntime‚1938  _runtime‚1938Ä~«ÄÃR TGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚32145914 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 _step‚196 JDDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak‚0 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ^LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚ 2024586762752 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 trainer/global_step‚2417 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚148 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 aODeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.allocated‚ 2017134329271 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853513 ICDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚45642512 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15975 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492829184 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24399 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 ]IDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.freed‚155559564065280 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 2125953646592 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955585024 L?DeviceStatsMonitor.on_validation_batch_end/all¿U|WC"ocation.all.freed‚80759915 SFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚45626537 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚11646868 aMDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.allocated‚155559701253120 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 YKDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.current‚ 137187840 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚138834121403831 BDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24399 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 ]IDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.freed‚155559564065280 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 2125953646592 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955585024 L?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚80759915 SFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚45626537 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚11646868 aMDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.allocated‚155559701253120 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 YKDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.current‚ 137187840 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚138834121403831 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 trainer/global_step‚2417 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚148 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 aODeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.allocated‚ 2017134329271Ĥ5’Æ‚(:‡ òæÔ‹¯¥ý gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚188.194' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6388 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚03‰ÅJÇ‚(:¿ òæÔ¢½®€ disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚6833.51904296875 disk.sda2.out‚0 disk.rbd1.out‚ 10.39453125 disk.drbd1716.out‚0$ memory_percent‚98.70960712432861$ disk./.usageGB‚298.2522850036621' disk.nvme0c0n1.out‚6833.51904296875 disk.sda3.out‚ 73.70703125 disk.drbd2266.out‚0 disk.sda.in‚0 disk.sda2.in‚0 proc.memory.rssMB‚ 2635.1875 proc.cpu.threads‚17 network.sent‚ 4.619751e+06 disk.sda.out‚ 73.70703125 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚6833.51904296875 cpu‚25.375432112180416 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 73.70703125 disk.drbd1716.in‚0 disk.drbd2265.in‚0* proc.memory.percent‚16.083908081054688 network.recv‚726566) disk./.usagePercent‚20.01596109693125 disk.drbd2080.out‚0( proc.memory.availableMB‚ 211.41796875 disk.rbd0.out‚0 disk.dm-1.in‚0±8×á2È- _wandbruntime‚1951  _runtime‚1951Ö˜áw7.ÉЦ 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9896824955940247 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1560884267091751 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.01684636063873768 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12005887925624847 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.2740781009197235 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 particle/accuracy_5_of_6‚1 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.13333334028720856 )loss/val_total_loss‚56.79864862819631 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9577092528343201 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3328947424888611 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17712950706481934 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.10935673117637634 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 +jet/accuracy_3_of_5‚0.06227599456906319 +"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.25 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9908045530319214 6 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.320175439119339 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_3‚1 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6279280185699463 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7424242496490479 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 ;"Purity/1FRt0SRqqt2FBt/event_purity‚0.055789753794670105 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.32683834433555603 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.4836917519569397 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.3125 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9982308745384216 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 +"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.25 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9937306046485901 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.08903777599334717 , Purity/1FRt0SRqqt3FBt/FRt_purity‚0.53125 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.27872589230537415 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.08584482222795486 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.11751185357570648 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.08156028389930725 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 *jet/accuracy_1_of_1‚0.3827546536922455 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.24259261786937714 ("Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.21231234073638916 +"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.25 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 +"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.25 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.3189655542373657 particle/accuracy_1_of_3‚1 (particle/accuracy‚0.8223533630371094 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.16277675330638885 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9897289872169495 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.29107141494750977 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.15898586809635162 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.12653490900993347 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.25925925374031067 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.2670991122722626 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.07851403206586838 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 particle/accuracy_4_of_4‚1 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 7 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.1557094156742096 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 +jet/accuracy_2_of_6‚0.24358972907066345 jet/accuracy_2_of_7‚0 8"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.419971227645874 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 *jet/accuracy_1_of_3‚0.7547053694725037 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.4375 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9862953424453735 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 6 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.360584557056427 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9817898869514465 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 :"Purity/2FRt*SRqqt0FBt/event_purity‚0.00133200129494071 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.11531531065702438 7 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.1781609207391739 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_6‚1 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 jet/accuracy_7_of_7‚0 +"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.25 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.29987820982933044 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.21875 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 & Purity/4FRt1SRqqt0FBt/FRt_purity‚0 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.16447627544403076 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_2_of_5‚1 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.07959362119436264 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9979325532913208 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.24841925501823425 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 *jet/accuracy_1_of_6‚0.9230769276618958 ,"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.375 particle/accuracy_4_of_7‚1 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 ;"Purity/2FRt*SRqqt2FBt/event_purity‚0.011627906933426857 particle/accuracy_1_of_1‚1 particle/accuracy_3_of_5‚1 particle/accuracy_4_of_5‚1 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 7 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.9666666388511658 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 'particle/f_score‚0.3432246148586273 jet/accuracy_6_of_6‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.18487651646137238 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 8"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.285748690366745 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.11414140462875366 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.19135801494121552 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 particle/accuracy_6_of_6‚1 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9915684461593628 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9166666269302368 :"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.38333332538604736 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.8859648704528809 ("Purity/*FRt*SRqqt4FBt/event_purity‚0 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.6666666865348816 & Purity/4FRt1SRqqt*FBt/FRt_purity‚0 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.2748519778251648 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 :"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.40927058458328247 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 6 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.095238097012043 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.0833333358168602 particle/accuracy_5_of_7‚1 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.30285295844078064 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15347543358802795 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.28118568658828735 8 Purity/2FRt1SRqqt1FBt/FRt_puritZžú` ey‚0.27589288353919983 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.16284659504890442 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.4624383747577667 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 particle/accuracy_2_of_4‚1 *"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5 & Purity/*FRt1SRqqt3FBt/FRt_purity‚0 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.2774931788444519 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 & Purity/1FRt*SRqqt4FBt/FRt_purity‚0 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚0 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚1 _runtime‚1951.842326698 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.28843653202056885 :"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.25458088517189026 ("Purity/0FRt0SRqqt4FBt/event_purity‚0 particle/accuracy_1_of_7‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.08144112676382065 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.4584745764732361 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.4132075905799866 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.003710575168952346 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.12599147856235504 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.15696071088314056 *jet/accuracy_2_of_4‚0.2783171832561493 epoch‚1 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9800630807876587 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8191796541213989 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.27118054032325745 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.11515454202890396 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.20000000298023224 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.36876705288887024 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 8 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.24242424964904785 particle/accuracy_5_of_5‚1 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9850995540618896 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.1459970772266388 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.2771317958831787 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 7 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.1209770143032074 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 particle/accuracy_3_of_7‚1 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.26332125067710876 /"Purity/0FRt0SRqqt3FBt/event_purity‚0.140625 +"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.25 *"Purity/0FRt2SRqqt3FBt/SRqqt_purity‚0.5 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.006952384952455759 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 _step‚197 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 particle/accuracy_3_of_6‚1 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.3155086040496826 jet/accuracy_5_of_7‚0 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.06753063946962357 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.07932677119970322 ,jet/accuracy_5_of_5‚0.001792114693671465 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 & Purity/4FRt*SRqqt0FBt/FRt_purity‚0 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.4509851038455963 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 9"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.6499999761581421 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 :"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.44156888127326965 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9737603068351746 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9817553758621216 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 :"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.27160218358039856 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.2916666567325592 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 7 Purity/*FRt*SRqqt4FBt/FBt_purity‚0.8333333134651184 & Purity/*FRt*SRqqt4FBt/FRt_purity‚0 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚0 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.15541669726371765 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.25988513231277466 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1677214652299881 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.4486111104488373 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.31512051820755005 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.4021727740764618 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 +jet/accuracy_3_of_4‚0.03979268670082092 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.07777777314186096 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.3783493638038635 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7121212482452393 particle/accuracy_7_of_7‚1 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.3541666567325592 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.3541666567325592 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9829146265983582 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.3155086040496826 ;"Purity/1FRt*SRqqt*FBt/event_purity‚0.049507029354572296 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 % _timestamp‚1.7884468363685088e+09 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 *jet/accuracy_2_of_3‚0.1884160339832306 6 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.229166641831398 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.1031746044754982 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9909297227859497 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9951155185699463 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.10221833735704422 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.15743890404701233 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.2153199017047882 ,jet/accuracy_4_of_5‚0.003808243665844202 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.4285714328289032 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9434522986412048 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.3400000035762787 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 +particle/specificity‚0.9194594025611877 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.24691356718540192 +"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.25 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.25153324007987976 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,jet/accuracy_3_of_3‚0.014781726524233818 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.13189654052257538 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.3159722089767456 jet/accuracy_5_of_6‚0 +"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.25 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 -"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.4375 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.4444444477558136 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.4875960052013397 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 particle/accuracy_2_of_2‚1 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.27910083532333374 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.20000000298023224 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.16415275633335114 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.06060606241226196 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9877104759216309 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9871304631233215 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 jet/accuracy_4_of_7‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9945933222770691 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 -jet/accuracy_4_of_4‚0.0009920635493472219 particle/accuracy_1_of_4‚1 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3642936646938324 7 Purity/2FRt0SRqqt1FBt/FBt_purity‚0.9940476417541504 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.2361111044883728 8 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.20000000298023224 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.007775804027915001 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 jet/accuracy_6_of_7‚0 particle/accuracy_6_of_7‚1 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.4146277904510498 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.009189971722662449 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_6‚1 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 jet/accuracy_1_of_7‚1 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.10710455477237701 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.0555555559694767 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.33315354585647583 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 <"Purity/2FRt1SRqqt*FBt/event_purity‚0.0022675737272948027 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0833333358168602 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.13650690019130707 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ;"Purity/0FRt2SRqqt0FBt/event_purity‚0.017252236604690552 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.2977646589279175 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9952381253242493 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9847283959388733 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.3027488589286804 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.4618590176105499 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.1041666716337204 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9246033430099487 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.3338245749473572 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.21875 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.39908352494239807 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.07959362119436264 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.023271219804883003 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9614583849906921 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.2839505970478058 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.42105263471603394 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.1692323386669159 particle/accuracy_2_of_3‚1 particle/accuracy_1_of_6‚1 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.0833333358168602 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9877192974090576 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚0 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 :"Purity/*FRt1SRqqt2FBt/event_purity‚0.01939324103295803 particle/accuracy_3_of_4‚1 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0833333358168602 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 ( Purity/*FRt0SRqqt3FBt/FRt_purity‚0.5 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.12972214818000793 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9726585149765015 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.2728351354598999 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.26633983850479126 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.26914656162261963 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 :"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.32665345072746277 *jet/accuracy_1_of_2‚0.6289793252944946 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.10526315867900848 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9078015089035034 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.2990550696849823 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9879103302955627 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.1643770933151245 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.02083333395421505 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.2750000059604645 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13610665500164032 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 *jet/accuracy_1_of_4‚0.8151216506958008 *jet/accuracy_1_of_5‚0.8528289794921875 7validation_average_jet_accuracy‚0.35370051860809326 particle/accuracy_2_of_7‚1 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9915463924407959 9"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.2618735730648041 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.1041666716337204 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 6 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.977633535861969 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 :"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.25977009534835815 particle/accuracy_1_of_5‚1 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9700000286102295 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.16658897697925568 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 jet/accuracy_3_of_7‚0 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚0 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.3128212094306946 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.4248141348361969 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9876779913902283 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.003907619975507259 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.008052513934671879 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚0 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.13917534053325653 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9928571581840515 ;"Purity/2FRt1SRqqt2FBt/event_purity‚0.022727273404598236 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.3397114872932434 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9980338215827942 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9941308498382568 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.3111010193824768 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.38738906383514404 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 ("Purity/*FRt0SRqqt4FBt/event_purity‚0 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6499837636947632 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.32827574014663696 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.004005920607596636 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 *"Purity/*FRt2SRqqt3FBt/SRqqt_purity‚0.5 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.2627559006214142 ;"Purity/0FRt2SRqqt*FBt/event_purity‚0.017900584265589714 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9982308745384216 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 +jet/accuracy_2_of_5‚0.33600232005119324 ("Purity/0FRt*SRqqt4FBt/event_purity‚0 ;"Purity/1FRt*SRqqt2FBt/event_purity‚0.046643417328596115 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.1595918983221054 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ,particle/sensitivity‚0.29747945070266724 8 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.16111111640930176 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.2222222238779068 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.008484505116939545 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.34457847476005554 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.3045177161693573 :"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.47672808170318604 jet/accuracy_4_of_6‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.14120370149612427 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9642857313156128 trainer/global_step‚2417 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.1726190447807312 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.39666667580604553 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.0625 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9846909642219543 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.2400793731212616 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.3207338750362396 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.007395096588879824 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 ;"Purity/1FRt*SRqqt0FBt/event_purity‚0.035749439150094986 + Purity/0FRt*SRqqt4FBt/FBt_purity‚0.8125 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.975680410861969 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.14556878805160522 + Purity/0FRt0SRqqt4FBt/FBt_purity‚0.8125 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ("Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0 & Purity/1FRt1SRqqt3FBt/FRt_purity‚0 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.18924935162067413 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.30396145582199097 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.13229399919509888 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.3994332551956177 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.2591717839241028 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.28041666746139526 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.3888888955116272 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8500000238418579 & Purity/4FRt*SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.16399671137332916 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.264973521232605 +jet/accuracy_3_of_6‚0.13461539149284363 ,"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.375 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.4104166626930237 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 9"Purity/0FRt1SRqqt2FBt/event_purity‚0.0258620698004961 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.27660587430000305ùò%ÒÊÕ¦ & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 7 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.1557094156742096 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 +jet/accuracy_2_of_6‚0.24358972907066345 jet/accuracy_2_of_7‚0 8"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.419971227645874 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 *jet/accuracy_1_of_3‚0.7547053694725037 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.4375 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9862953424453735 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 6 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.360584557056427 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9817898869514465 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 :"Purity/2FRt*SRqqt0FBt/event_purity‚0.00133200129494071 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.11531531065702438 7 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.1781609207391739 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_6‚1 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 jet/accuracy_7_of_7‚0 +"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.25 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.29987820982933044 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.21875 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 & Purity/4FRt1SRqqt0FBt/FRt_purity‚0 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.16447627544403076 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_2_of_5‚1 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.07959362119436264 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9979325532913208 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.24841925501823425 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 *jet/accuracy_1_of_6‚0.9230769276618958 ,"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.375 particle/accuracy_4_of_7‚1 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 ;"Purity/2FRt*SRqqt2FBt/event_purity‚0.011627906933426857 particle/accuracy_1_of_1‚1 particle/accuracy_3_of_5‚1 particle/accuracy_4_of_5‚1 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 7 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.9666666388511658 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 'particle/f_score‚0.3432246148586273 jet/accuracy_6_of_6‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.18487651646137238 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 8"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.285748690366745 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.11414140462875366 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.19135801494121552 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 particle/accuracy_6_of_6‚1 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9915684461593628 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9166666269302368 :"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.38333332538604736 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.8859648704528809 ("Purity/*FRt*SRqqt4FBt/event_purity‚0 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.6666666865348816 & Purity/4FRt1SRqqt*FBt/FRt_purity‚0 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.2748519778251648 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 :"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.40927058458328247 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 6 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.095238097012043 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.0833333358168602 particle/accuracy_5_of_7‚1 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.30285295844078064 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15347543358802795 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.28118568658828735 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.27589288353919983 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.16284659504890442 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.4624383747577667 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 particle/accuracy_2_of_4‚1 *"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5 & Purity/*FRt1SRqqt3FBt/FRt_purity‚0 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.04954117164]ÆïŠx01577 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.2774931788444519 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 & Purity/1FRt*SRqqt4FBt/FRt_purity‚0 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚0 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚1 _runtime‚1951.842326698 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.28843653202056885 :"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.25458088517189026 ("Purity/0FRt0SRqqt4FBt/event_purity‚0 particle/accuracy_1_of_7‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.08144112676382065 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.4584745764732361 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.4132075905799866 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.003710575168952346 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.12599147856235504 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.15696071088314056 *jet/accuracy_2_of_4‚0.2783171832561493 epoch‚1 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9800630807876587 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8191796541213989 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.27118054032325745 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.11515454202890396 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.20000000298023224 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.36876705288887024 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 8 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.24242424964904785 particle/accuracy_5_of_5‚1 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9850995540618896 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.1459970772266388 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.2771317958831787 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 7 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.1209770143032074 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 particle/accuracy_3_of_7‚1 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.26332125067710876 /"Purity/0FRt0SRqqt3FBt/event_purity‚0.140625 +"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.25 *"Purity/0FRt2SRqqt3FBt/SRqqt_purity‚0.5 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.006952384952455759 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 _step‚197 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 particle/accuracy_3_of_6‚1 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.3155086040496826 jet/accuracy_5_of_7‚0 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.06753063946962357 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.07932677119970322 ,jet/accuracy_5_of_5‚0.001792114693671465 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 & Purity/4FRt*SRqqt0FBt/FRt_purity‚0 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.4509851038455963 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 9"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.6499999761581421 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 :"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.44156888127326965 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9737603068351746 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9817553758621216 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 :"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.27160218358039856 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.2916666567325592 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 7 Purity/*FRt*SRqqt4FBt/FBt_purity‚0.8333333134651184 & Purity/*FRt*SRqqt4FBt/FRt_purity‚0 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚0 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.15541669726371765 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.25988513231277466 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1677214652299881 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.4486111104488373 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.31512051820755005 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.4021727740764618 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 +jet/accuracy_3_of_4‚0.03979268670082092 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.07777777314186096 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.3783493638038635 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7121212482452393 particle/accuracy_7_of_7‚1 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.3541666567325592 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.3541666567325592 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9829146265983582 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.3155086040496826 ;"Purity/1FRt*SRqqt*FBt/event_purity‚0.049507029354572296 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 % _timestamp‚1.7884468363685088e+09 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 *jet/accuracy_2_of_3‚0.1884160339832306 6 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.229166641831398 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.1031746044754982 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9909297227859497 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9951155185699463 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.10221833735704422 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.15743890404701233 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.2153199017047882 ,jet/accuracy_4_of_5‚0.003808243665844202 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.4285714328289032 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9434522986412048 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.3400000035762787 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 +particle/specificity‚0.9194594025611877 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.24691356718540192 +"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.25 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.25153324007987976 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,jet/accuracy_3_of_3‚0.014781726524233818 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.13189654052257538 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.3159722089767456 jet/accuracy_5_of_6‚0 +"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.25 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 -"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.4375 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.4444444477558136 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.4875960052013397 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 particle/accuracy_2_of_2‚1 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.27910083532333374 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.20000000298023224 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.16415275633335114 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.06060606241226196 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9877104759216309 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9871304631233215 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 jet/accuracy_4_of_7‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9945933222770691 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 -jet/accuracy_4_of_4‚0.0009920635493472219 particle/accuracy_1_of_4‚1 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3642936646938324 7 Purity/2FRt0SRqqt1FBt/FBt_purity‚0.9940476417541504 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.2361111044883728 8 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.20000000298023224 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.007775804027915001 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 jet/accuracy_6_of_7‚0 particle/accuracy_6_of_7‚1 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.4146277904510498 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.009189971722662449 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_6‚1 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 jet/accuracy_1_of_7‚1 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.10710455477237701 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.0555555559694767 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.33315354585647583 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 <"Purity/2FRt1SRqqt*FBt/event_purity‚0.0022675737272948027 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0833333358168602 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.13650690019130707 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ;"Purity/0FRt2SRqqt0FBt/event_purity‚0.017252236604690552 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.2977646589279175 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9952381253242493 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9847283959388733 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.3027488589286804 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.4618590176105499 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.1041666716337204 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9246033430099487 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.3338245749473572 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.21875 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.39908352494239807 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.07959362119436264 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.023271219804883003 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9614583849906921 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.2839505970478058 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.42105263471603394 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.1692323386669159 particle/accuracy_2_of_3‚1 particle/accuracy_1_of_6‚1 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.0833333358168602 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9877192974090576 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚0 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 :"Purity/*FRt1SRqqt2FBt/event_purity‚0.01939324103295803 particle/accuracy_3_of_4‚1 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0833333358168602 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 ( Purity/*FRt0SRqqt3FBt/FRt_purity‚0.5 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.12972214818000793 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9726585149765015 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.2728351354598999 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.26633983850479126 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.26914656162261963 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 :"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.32665345072746277 *jet/accuracy_1_of_2‚0.6289793252944946 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.10526315867900848 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9078015089035034 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.2990550696849823 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9879103302955627 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.1643770933151245 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.02083333395421505 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.2750000059604645 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13610665500164032 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 *jet/accuracy_1_of_4‚0.8151216506958008 *jet/accuracy_1_of_5‚0.8528289794921875 7validation_average_jet_accuracy‚0.35370051860809326 particle/accuracy_2_of_7‚1 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9915463924407959 9"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.2618735730648041 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.1041666716337204 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 6 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.977633535861969 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 :"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.25977009534835815 particle/accuracy_1_of_5‚1 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9700000286102295 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.16658897697925568 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 jet/accuracy_3_of_7‚0 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚0 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.3128212094306946 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.4248141348361969 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9876779913902283 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.003907619975507259 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.008052513934671879 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚0 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.13917534053325653 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9928571581840515 ;"Purity/2FRt1SRqqt2FBt/event_purity‚0.022727273404598236 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.3397114872932434 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9980338215827942 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9941308498382568 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.3111010193824768 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.38738906383514404 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 ("Purity/*FRt0SRqqt4FBt/event_purity‚0 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6499837636947632 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.32827574014663696 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.004005920607596636 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 *"Purity/*FRt2SRqqt3FBt/SRqqt_purity‚0.5 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.2627559006214142 ;"Purity/0FRt2SRqqt*FBt/event_purity‚0.017900584265589714 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9982308745384216 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 +jet/accuracy_2_of_5‚0.33600232005119324 ("Purity/0FRt*SRqqt4FBt/event_purity‚0 ;"Purity/1FRt*SRqqt2FBt/event_purity‚0.046643417328596115 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.1595918983221054 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ,particle/sensitivity‚0.29747945070266724 8 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.16111111640930176 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.2222222238779068 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.008484505116939545 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.34457847476005554 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.3045177161693573 :"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.47672808170318604 jet/accuracy_4_of_6‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.14120370149612427 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9642857313156128 trainer/global_step‚2417 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.1726190447807312 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.39666667580604553 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.0625 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9846909642219543 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.2400793731212616 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.3207338750362396 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.007395096588879824 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 ;"Purity/1FRt*SRqqt0FBt/event_purity‚0.035749439150094986 + Purity/0FRt*SRqqt4FBt/FBt_purity‚0.8125 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.975680410861969 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.14556878805160522 + Purity/0FRt0SRqqt4FBt/FBt_purity‚0.8125 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ("Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0 & Purity/1FRt1SRqqt3FBt/FRt_purity‚0 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.18924935162067413 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.30396145582199097 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.13229399919509888 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.3994332551956177 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.2591717839241028 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.28041666746139526 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.3888888955116272 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8500000238418579 & Purity/4FRt*SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.16399671137332916 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.264973521232605 +jet/accuracy_3_of_6‚0.13461539149284363 ,"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.375 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.4104166626930237 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 9"Purity/0FRt1SRqqt2FBt/event_purity‚0.0258620698004961 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.27660587430000305 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9896824955940247 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1560884267091751 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.01684636063873768 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12005887925624847 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.2740781009197235 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 particle/accuracy_5_of_6‚1 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.13333334028720856 )loss/val_total_loss‚56.79864862819631 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9577092528343201 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3328947424888611 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17712950706481934 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.10935673117637634 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 +jet/accuracy_3_of_5‚0.06227599456906319 +"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.25 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9908045530319214 6 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.320175439119339 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_3‚1 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6279280185699463 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7424242496490479 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 ;"Purity/1FRt0SRqqt2FBt/event_purity‚0.055789753794670105 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.32683834433555603 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.4836917519569397 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.3125 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9982308745384216 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 +"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.25 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9937306046485901 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.08903777599334717 , Purity/1FRt0SRqqt3FBt/FRt_purity‚0.53125 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.27872589230537415 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.08584482222795486 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.11751185357570648 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.08156028389930725 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 *jet/accuracy_1_of_1‚0.3827546536922455 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.24259261786937714 ("Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.21231234073638916 +"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.25 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 +"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.25 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.3189655542373657 particle/accuracy_1_of_3‚1 (particle/accuracy‚0.8223533630371094 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.16277675330638885 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9897289872169495 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.29107141494750977 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.15898586809635162 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.12653490900993347 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.25925925374031067 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.2670991122722626 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.07851403206586838 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 particle/accuracy_4_of_4‚1 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 ("Purity/1FRt1SRqqt4FBt/event_purity‚0ÅLèW“Ë‚(:ˆ ‘æÔ¶øŠý gpu.0.gpu‚24.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚134.419' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.8838 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ˆï\GÌ‚(:¼ ‘æÔàåÇÿ disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.dm-1.in‚0 disk.dm-1.out‚ 74.03515625* disk./.usagePercent‚20.048442564853058$ disk./.usageGB‚298.7362823486328 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.sda3.out‚ 74.03515625 disk.sda.out‚ 74.03515625 disk.sda2.in‚0 disk.rbd1.out‚ 10.39453125" proc.memory.rssMB‚ 2642.7109375 cpu‚25.374635058390282 network.recv‚732013 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.sda2.out‚0 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125( proc.memory.availableMB‚ 950.35546875) proc.memory.percent‚16.12982749938965 disk.sda1.out‚0 disk.sda3.in‚0 network.sent‚ 4.726684e+06 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125$ memory_percent‚94.19949054718018% disk.nvme0c0n1.out‚7852.841796875 disk.rbd0.in‚0 disk.sda1.in‚0# disk.nvme0n1.out‚7852.841796875 disk.drbd2265.out‚0 disk.dm-0.out‚7852.841796875RœŽ{2Í- _wandbruntime‚1979  _runtime‚1979ÀzŒÆÎÀ & lr-AdamW/pg1‚0.0001226693637806697 & lr-AdamW/pg2‚0.0001226693637806697 trainer/global_step‚2449 % _timestamp‚1.7884468638429458e+09 _runtime‚ 1979.31456645 _step‚198šˆ™„zÏÅ % _timestamp‚1.7884468638429458e+09 _runtime‚ 1979.31456645 _step‚198 & lr-AdamW/pg1‚0.0001226693637µNšÐQ806697 & lr-AdamW/pg2‚0.0001226693637806697 trainer/global_step‚2449Æ­´}2Ð- _wandbruntime‚1979  _runtime‚1979‹øCuØ'ÑÒO PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46314927 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35708352 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚155837281069568 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚426 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35708352 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚142722423289344 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2054093510656 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚144776516800000 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚141024345495413 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2046528539509 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2054749872640 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚138977816955904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚157994729879552 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚2449 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2054749872640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2054093510656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚142722423289344 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚424 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46314927 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚157994536023552 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2157424869888 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚142722441292288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46294890 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46294890 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚144776516800000 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚142722441292288 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32682006 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915904 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20836804 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11844776 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193856000 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚82023281 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32681580 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20836806 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚155837305009664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2047182977261 _runtime‚1979.315481969 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35708354 $ _timestamp‚1.788446863843672e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚144777191164928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚144777191164928 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚138977833995264 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚82003242 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2157254953984 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚144776516800000 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚142722441292288 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32682006 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915904 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20836804 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11844776 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193856000 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚82023281 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32681580 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20836806 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚155837305009664 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2047182977261 _runtime‚1979.315481969 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35708354 $ _timestamp‚1.788446863843672e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚144777191164928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚144777191164928 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚138977833995264 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚82003242 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2157254953984 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚155837281069568 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚426 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35708352 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚142722423289344 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2054093510656 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚144776516800000 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚141024345495413 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2046528539509 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2054749872640 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚138977816955904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚157994729879552 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 trainer/global_step‚2449 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2054749872640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2054093510656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚142722423289344 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚424 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0ÇíWÁ“Ó‚(:ˆ ‘æÔø»“þ gpu.0.gpu‚49.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚175.492' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.0984 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0tZEÔ‚(:º ‘æÔÍÓ®ÿ( proc.memory.availableMB‚ 950.35546875" proc.memory.rssMB‚ 2642.7109375) proc.memory.percent‚16.12982749938965) disk./.usagePercent‚20.04866170730237$ disk./.usageGB‚298.7395477294922 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda.in‚0 disk.sda1.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125# disk.nvme0n1.out‚7896.630859375 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.sda3.in‚0 disk.rbd1.out‚ 10.45703125 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd1716.out‚0 network.recv‚738878 disk.dm-0.out‚7896.630859375 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.out‚ 74.61328125 disk.drbd2265.out‚0 disk.drbd2080.in‚0$ memory_percent‚94.19949054718018 disk.sda.out‚ 74.61328125 disk.rbd1.in‚0 disk.sda3.out‚ 74.61328125 disk.sda2.out‚0 cpu‚25.37209213516077 network.sent‚ 4.851157e+06 disk.drbd2266.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd1716.in‚0% disk.nvme0c0n1.out‚7896.630859375€ë;2Õ- _wandbruntime‚1980  _runtime‚1980Ñ!Gùã&ÖÝM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2055569313792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11849668 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2055569313792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚158057038885376 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 trainer/global_step‚2449 _step‚200 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35722368 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚142778776328192 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚142778776328192 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚139032738125824 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46333407 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32694642 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚139032721086464 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46333407 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚144833689266688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚155898931807232 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚142778794331136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35722370 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2047999412433 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚144834363644928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚144834363644928 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚155898955747328 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46313344 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82055777 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32694211 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2054912938496 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚431 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚142778794331136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2158276980736 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2158107078144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚141080066060937 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2054912938496 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11849239 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46313344 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82055777 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚141080737538257 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2047344974473 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 _runtime‚1980.028764894 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35722370 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚82035712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚158057232728064 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82035712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7884468645568025e+09 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 trainer/global_step‚2449 _step‚200 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35722368 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚142778776328192 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚142778776328192 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚139032738125824 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46333407 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32694642 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚139032721086464 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46333407 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚144833689266688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚155898931807232 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚142778794331136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35722370 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2047999412433 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚144834363644928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚144834363644928 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚155898955747328 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46313344 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82055777 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32694211 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2054912938496 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚431 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚142778794331136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2158276980736 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2158107078144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚141080066060937 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2054912938496 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11849239 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚46313344 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82055777 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚141080737538257 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2047344974473 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 _runtime‚1980.028764894 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35722370 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚82035712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚429 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚158057232728064 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82035712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7884468645568025e+09 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2055569313792Èwv2Ø- _wandbruntime‚1980  _runtime‚1980ñPæUÙÏ 0loss/FBt4/assignment_loss‚0.9327306169240425 0loss/FBt2/assignment_loss‚0.4628286535740661 /loss/FRt2/detection_loss‚2.3912255908991837 /loss/FBt4/detection_loss‚3.8368069065941706 _step‚201 1loss/SRqqt1/assignment_loss‚2.840019295612971 .loss/FRt3/detection_loss‚2.277356138756109 .loss/FRt4/detection_loss‚2.565196159211072 1loss/SRqqt3/detection_loss‚1.4719197503996617 1loss/SRqqt4/detection_loss‚1.8670434105609144 .loss/FBt2/detection_loss‚2.830409300327301 .loss/FRt1/assignment_loss‚4.70516037940979 1loss/FBt3/assignment_loss‚0.20154080866683868 %loss/total_loss‚53.37813775533279 0loss/FRt2/assignment_loss‚3.2927376772906327 0loss/FRt3/assignment_loss‚3.3118950197863026 /loss/FBt1/assignment_loss‚0.282016522642889 /loss/FBt1/detection_loss‚3.2697681598365307 trainer/global_step‚2449 0loss/SRqqt1/detection_loss‚2.869652173916499 0loss/SRqqt2/detection_loss‚1.987719918290774 /loss/FBt3/detection_loss‚1.0473599232900053 /loss/FRt1/detection_loss‚2.8359324586006904 % _timestamp‚1.7884468645583086e+09 0loss/FRt4/assignment_loss‚3.1854926225813953 2loss/SRqqt3/assignment_loss‚1.5915702444247224 epoch‚2 _runtime‚ 1980.0297958 2loss/SRqqt2/assignment_loss‚1.6658873455155463 2loss/SRqqt4/assignment_loss‚1.6558686782206808—‰9~ZÚÔ /loss/FRt1/detection_loss‚2.8359324586006904 % _timestamp‚1.7884468645583086e+09 0loss/FRt4/assignment_loss‚3.1854926225813953 2loss/SRqqt3/assignment_loss‚1.5915702444247224 epoch‚2 _runtime‚ 1980.0297958 2loss/SRqqt2/assignment_loss‚1.6658873455155463 2loss/SRqqt4/assignment_loss‚1.6558686782206808 0loss/FBt4/assignment_loss‚0.9327306169240425 0loss/FBt2/assignment_loss‚0.4628286535740661 /loss/FRt2/detection_loss‚2.3912255908991837 /loss/FBt4/detection_loss‚3.8368069065941706 _step‚201 1loss/SRqqt1/assignment_loss‚2.840019295612971 .loss/FRt3/detection_loss‚2.277356138756109 .loss/FRt4/detection_loss‚2.565196159211072 1loss/SRqqt3/detection_loss‚1.4719197503996617 1loss/SRqqt4/detection_loss‚1.8670434105609144 .loss/FBt2/detection_loss‚2.830409300327301 .loss/FRt1/assignment_loss‚4.70516037940979 1loss/FBt3/assignment_loss‚0.20154080866683868 %loss/total_loss‚53.37813775533279 0loss/FRt2/assignment_loss‚3.2927376772906327 0loss/FRt3/assignment_loss‚3.3118950197863026 /loss/FBt1/assignment_loss‚0.282016522642889 /loss/FBt1/detection_loss‚3.2697681598365307 trainer/global_step‚2449 0loss/SRqqt1/detection_loss‚2.869652173916499 0loss/SRqqt2/detection_loss‚1.987719918290774 /loss/FBt3/detection_loss‚1.0473599232900053ɘ­žÛ‚(:“ Ÿ‘æÔϽòü gpu.0.gpu‚49.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.268' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.653600000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¸eÄIÜ‚(:¾ Ÿ‘æÔ¦Šôÿ disk.sda.in‚0 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd1716.out‚0 network.sent‚ 4.947277e+06 disk.rbd0.out‚0 disk.sda.out‚ 75.0703125 disk.drbd2265.in‚0) proc.memory.percent‚16.12982749938965 disk.sda3.in‚0 disk.rbd1.out‚ 10.58203125% disk.nvme0n1.out‚7946.70654296875$ memory_percent‚94.19996738433838( proc.memory.availableMB‚ 950.27734375 disk.sda3.out‚ 75.0703125 disk.sda2.in‚0 disk.drbd2080.out‚0 network.recv‚743932 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 75.0703125 disk.drbd2080.in‚0 cpu‚25.36956817931421 proc.cpu.threads‚17$ disk./.usageGB‚298.7456283569336' disk.nvme0c0n1.out‚7946.70654296875 disk.rbd0.in‚0" disk.dm-0.out‚7946.70654296875 disk.drbd2265.out‚0* disk./.usagePercent‚20.049069783311946 disk.dm-0.in‚0.078125" proc.memory.rssMB‚ 2642.7109375 disk.drbd2266.out‚0AÖ¡}Ý‚(:’ ®‘æÔÕ€“ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.17' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.833999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0\UÛWKÞ‚(:À ®‘æÔ’¤Êÿ network.sent‚ 5.045742e+06 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0* disk./.usagePercent‚20.049569766026316 disk.sda3.in‚0 disk.drbd2266.in‚0" disk.dm-0.out‚8000.93701171875 disk.rbd0.in‚0 disk.rbd1.out‚ 10.64453125% disk.nvme0n1.out‚8000.93701171875 disk.dm-1.in‚0% proc.memory.availableMB‚ 950.28125 disk.sda3.out‚ 75.57421875 disk.sda.in‚0 disk.sda.out‚ 75.57421875 disk.sda1.in‚0) proc.memory.percent‚16.12982749938965 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.out‚0 disk.drbd2265.in‚0 network.recv‚750457 disk.sda2.in‚0 disk.dm-1.out‚ 75.57421875 disk.drbd1716.in‚0$ memory_percent‚94.19994354248047' disk.nvme0c0n1.out‚8000.93701171875 disk.rbd1.in‚0% disk./.usageGB‚298.75307846069336 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125" proc.memory.rssMB‚ 2642.7109375 cpu‚25.367225542425604$8]¯2ß- _wandbruntime‚2016  _runtime‚2016IÁ¶Éàà ' lr-AdamW/pg2‚0.00012234276269446008 trainer/global_step‚2499 % _timestamp‚1.7884469013188016e+09 _runtime‚2016.790389175 _step‚202 ' lr-AdamW/pg1‚0.00012234276269446008Š+S§ÎáÈ _runtime‚2016.790389175 _step‚202 ' lr-AdamW/pg1‚0.00012234276269446008 ' lr-AdamW/pg2‚0.00012234276269446008 trainer/global_step‚2499 % _timestamp‚1.7884469013188016e+09Ê`® £2â- _wandbruntime‚2016  _runtime‚2016ÞqëGÙ'ãÓO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2095735678464 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚158919841892864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2200298192896 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚143811058839933 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚143810387362821 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36409154 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚145540075231744 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47220942 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2095735678464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21245204 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21245206 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12070986 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36409154 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884469013194892e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2095079316480 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚158919817952768 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚147635154548224 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚145540093234688å€ ÎäØO _step‚203 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36409154 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚145540075231744 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47220942 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2095735678464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21245204 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21245206 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12070986 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36409154 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884469013194892e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2095079316480 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚158919817952768 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚147635154548224 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚145540093234688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2095735678464 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚158919841892864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2200298192896 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚143811058839933 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚143810387362821 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚147693001393152 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83682629 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12075582 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚460 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21253374 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83682629 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚161182647244288 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚145596446273536 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47239396 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚158981468690432 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚145596428270592 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚145596446273536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2096555119616 _step‚204 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12075124 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚141777927614464 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21253372 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2088834751841 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36423170 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33328956 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47239396 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36423168 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2088180313881 % _timestamp‚1.7884469020030098e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83662564 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33328496 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47259459 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚147693001393152 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚143866107928345 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚158981492630528 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚458 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚161182453401600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2095898744320 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2096555119616 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚2499 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2095898744320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2$¶Ãè&çâM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚2499 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2095898744320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚147693001393152 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83682629 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12075582 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚460 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21253374 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83682629 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚161182647244288 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚145596446273536 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47239396 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚158981468690432 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚145596428270592 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚145596446273536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2096555119616 _step‚204 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12075124 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚141777927614464 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21253372 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2088834751841 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36423170 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33328956 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47239396 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36423168 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2088180313881 % _timestamp‚1.7884469020030098e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83662564 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33328496 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47259459 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚147693001393152 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚143866107928345 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚158981492630528 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚458 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚161182453401600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2095898744320 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2096555119616 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063ÌÉæÕ2è- _wandbruntime‚2017  _runtime‚2017öÏj1VéÐ /loss/FBt3/detection_loss‚0.8201199697126469 epoch‚2 .loss/FRt1/assignment_loss‚4.42024212413364 0loss/FBt1/assignment_loss‚0.4982402473686945 -loss/FRt2/detection_loss‚2.37431148712228 .loss/FBt1/detection_loss‚3.383331596851349 _runtime‚2017.476003964 0loss/FRt3/assignment_loss‚2.7302027012481065 0loss/FBt4/assignment_loss‚0.6628478728234768 .loss/FRt1/detection_loss‚3.056937897646869 1loss/SRqqt2/detection_loss‚1.8290212885388788 %loss/total_loss‚54.66649213681233 0loss/FRt4/assignment_loss‚3.6405086609033437 2loss/SRqqt4/assignment_loss‚1.9313009979669005 1loss/SRqqt1/detection_loss‚3.1876767724752426 2loss/SRqqt1/assignment_loss‚2.6868609438339868 2loss/SRqqt2/assignment_loss‚1.9189993514204926 1loss/FBt3/assignment_loss‚0.20302779242887437 1loss/SRqqt3/assignment_loss‚1.515348372956117 1loss/SRqqt3/detection_loss‚1.2187868364651997 1loss/SRqqt4/detection_loss‚1.6634404475335032 /loss/FRt2/assignment_loss‚4.046692496392785 .loss/FBt2/detection_loss‚2.336026208475232 /loss/FBt4/detection_loss‚5.3822596073150635 trainer/global_step‚2499 0loss/FBt2/assignment_loss‚0.5193453063256879 /loss/FRt3/detection_loss‚2.0025824818943367 /loss/FRt4/detection_loss‚2.6383806749796257 $ _timestamp‚1.788446902004479e+09 _step‚205â ˆ[êÕ 2loss/SRqqt4/assignment_loss‚1.9313009979669005 1loss/SRqqt1/detection_loss‚3.1876767724752426 2loss/SRqqt1/assignment_loss‚2.6868609438339868 2loss/SRqqt2/assignment_loss‚1.9189993514204926 1loss/FBt3/assignment_loss‚0.20302779242887437 1loss/SRqqt3/assignment_loss‚1.515348372956117 1loss/SRqqt3/detection_loss‚1.2187868364651997 1loss/SRqqt4/detection_loss‚1.6634404475335032 /loss/FRt2/assignment_loss‚4.046692496392785 .loss/FBt2/detection_loss‚2.336026208475232 /loss/FBt4/detection_loss‚5.3822596073150635 trainer/global_step‚2499 0loss/FBt2/assignment_loss‚0.5193453063256879 /loss/FRt3/detection_loss‚2.0025824818943367 /loss/FRt4/detection_loss‚2.6383806749796257 $ _timestamp‚1.788446902004479e+09 _step‚205 /loss/FBt3/detection_loss‚0.8201199697126469 epoch‚2 .loss/FRt1/assignment_loss‚4.42024212413364 0loss/FBt1/assignment_loss‚0.4982402473686945 -loss/FRt2/detection_loss‚2.37431148712228 .loss/FBt1/detection_loss‚3.383331596851349 _runtime‚2017.476003964 0loss/FRt3/assignment_loss‚2.7302027012481065 0loss/FBt4/assignment_loss‚0.6628478728234768 .loss/FRt1/detection_loss‚3.056937897646869 1loss/SRqqt2/detection_loss‚1.8290212885388788 %loss/total_loss‚54.66649213681233 0loss/FRt4/assignment_loss‚3.6405086609033437ÍË‘g‡žë‚(:“ ½‘æÔµÎþý gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚136.382' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.276400000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0$¸úfAì‚(:¶ ½‘æÔ¸·³€ network.sent‚ 5.051019e+06* disk./.usagePercent‚20.049835501846857$ disk.nvme0c0n1.out‚ 8056.33203125 disk.sda3.out‚ 76.2734375 disk.rbd1.in‚0 disk.dm-1.out‚ 76.2734375 disk.drbd1716.out‚0$ disk./.usageGB‚298.7570381164551 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚0 disk.drbd1716.in‚0$ memory_percent‚94.20597553253174 network.recv‚753721 disk.sda2.out‚0( proc.memory.availableMB‚ 949.29296875 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.dm-0.out‚ 8056.33203125 proc.cpu.threads‚17 disk.drbd2080.in‚0" proc.memory.rssMB‚ 2642.7109375 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 cpu‚25.364888748144942 disk.sda3.in‚0 disk.sda.in‚0 disk.sda.out‚ 76.2734375 disk.rbd1.out‚ 10.64453125" disk.nvme0n1.out‚ 8056.33203125 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.drbd2265.in‚0) proc.memory.percent‚16.12982749938965ªï þží‚(:“ Ì‘æÔíðý gpu.0.gpu‚49.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.191' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.038199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0B½ @î‚(:µ Ì‘æÔÙÆ®€) disk./.usagePercent‚20.04773137592294 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 76.68359375 disk.sda3.in‚0 disk.rbd1.out‚ 10.6767578125 disk.drbd2265.out‚0" proc.memory.rssMB‚ 2642.7109375) proc.memory.percent‚16.12982749938965 cpu‚25.362727585758705 disk.drbd2080.out‚0 disk.sda.in‚0 disk.drbd2265.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0$ memory_percent‚94.20785903930664 disk.rbd0.out‚0& proc.memory.availableMB‚ 948.984375# disk.nvme0c0n1.out‚ 8105.3203125 disk.drbd2266.out‚0 disk.sda.out‚ 76.68359375 disk.drbd1716.out‚0 proc.cpu.threads‚17$ disk./.usageGB‚298.7256851196289 disk.drbd2080.in‚0 disk.sda3.out‚ 76.68359375 disk.rbd0.in‚0 disk.dm-1.in‚0 network.sent‚ 5.160023e+06 network.recv‚760321 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.dm-0.out‚ 8105.3203125 disk.sda2.out‚0! disk.nvme0n1.out‚ 8105.3203125 disk.drbd1716.in‚0Í¿™ð2ï- _wandbruntime‚2053  _runtime‚2053áé‡ Éðà ' lr-AdamW/pg2‚0.00012200386611412118 trainer/global_step‚2549 % _timestamp‚1.7884469380009537e+09 _runtime‚2053.472546131 _step‚206 ' lr-AdamW/pg1‚0.00012200386611412118[ÑQéÎñÈ % _timestamp‚1.7884469380009537e+09 _runtime‚2053.472546131 _step‚206 ' lr-AdamW/pg1‚0.00012200386611412118 ' lr-AdamW/pg2‚0.00012200386611412118 trainer/global_step‚2549Î2—j2ò- _wandbruntime‚2053  _runtime‚2053›q}èè óÓO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚432 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚146596429230229 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21653606 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 % _timestamp‚1.7884469380016587e+09 _runtime‚2053.473470195 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37109954 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚162002354835968 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48146994 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚150493792296448 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48167031 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚150493792296448 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2136065122304 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚148357745177088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2136721484288 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚85276985 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181273088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2136065122304 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21653604 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 trainer/global_step‚2549 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37109952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2136721484288 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚85256946 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48167031 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚434 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37109952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12296551 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚164245694842368 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _step‚207 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚148357727174144 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33950157 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚144468247051264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚144468247051264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚432 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚146596429230229 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21653606 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 % _timestamp‚1.7884469380016587e+09 _runtime‚2053.473470195 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37109954 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚162002354835968 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48146994 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚150493792296448 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48167031 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚150493792296448 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2136065122304 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚148357745177088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2136721484288 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚85276985 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181273088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2136065122304 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21653604 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 trainer/global_step‚2549 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37109952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2136721484288 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚85256946 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48167031 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚434 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37109952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12296551 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚164245694842368 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _step‚207Ïùe s2õ- _wandbruntime‚2054  _runtime‚2054!¶ã&öÝM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12301002 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 trainer/global_step‚2549 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37123968 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37123968 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚162064029513728 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚144523151181824 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚85289416 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48165448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚85309481 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7884469386933215e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37123970 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48185511 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚148414080212992 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2136884550144 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33962332 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚146652821273073 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2244022709248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37123970 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚208 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85289416 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚146652149795753 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2129670091249 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85309481 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12300560 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513984 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48185511 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚148414098215936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844Mãù÷âM P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚85289416 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48165448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚85309481 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7884469386933215e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37123970 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48185511 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚148414080212992 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2136884550144 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33962332 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚146652821273073 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2244022709248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37123970 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚208 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85289416 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚146652149795753 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2129670091249 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85309481 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.oõ5Ádïn_train_batch_end/inactive_split.small_pool.freed‚12300560 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513984 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48185511 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚148414098215936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12301002 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 trainer/global_step‚2549 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37123968 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37123968 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚162064029513728 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚144523151181824 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚151175379116544 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86903837 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49073046 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚167370970259456 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163624448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚153353104409600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22062006 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2090.532283111 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12522091 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚151175397119488 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚153352430044672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚151175397119488 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22062004 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2286078540288 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2169034557733 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37810752 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2177707290112 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34584514 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚2599 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2177050928128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2177050928128 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚147213453579264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564544 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37810752 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚151175379116544 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49093083 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚167371157824000 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34584095 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚165084891719168 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2169688995485A³t'†ØO H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚151175379116544 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86903837 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49073046 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚167370970259456 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163624448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚153353104409600 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22062006 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2090.532283111 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12522091 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚151175397119488 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚153352430044672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚151175397119488 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22062004 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2286078540288 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2169034557733 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37810752 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2177707290112 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34584514 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚2599 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2177050928128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2177050928128 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚147213453579264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564544 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37810752 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚151175379116544 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49093083 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚167371157824000 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34584095 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚165084891719168 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2169688995485 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_ÅËìjpool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944Óâ®Ó62‡- _wandbruntime‚2091  _runtime‚2091©ìÁWâ&ˆÜM >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚153409602511360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49111563 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12526957 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2286941476864 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2170505430657 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚153410276889600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37824770 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚147268340670464 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚418 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86936333 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37824768 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12526539 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚167433671484928 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86916268 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22070174 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49091500 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚420 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 _step‚212 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚149438191663161 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚153410276889600 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2287105088000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34597131 trainer/global_step‚2599 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚151231732155392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34596711 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551232 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37824768 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49111563 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚151231750158336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86936333 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚167433483933696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚165146542456832 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2178526731264 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2169850992697 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚151231750158336 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22070172 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928÷R8=ç&‰áM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86936333 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37824768 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12526539 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚167433671484928 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86916268 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22070174 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49091500 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚420 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 _step‚212 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚149438191663161 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚153410276889600 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2287105088000 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34597131 trainer/global_step‚2599 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚151231732155392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34596711 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551232 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37824768 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49111563 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚151231750158336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86936333 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚167433483933696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚165146542456832 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2178526731264 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2169850992697 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚151231750158336 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22070172 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚153409602511360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49111563 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12526957 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2286941476864 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2170505430657 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚153410276889600 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37824770 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚147268340670464 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚418Ô`ZŸH2Š- _wandbruntime‚2091  _runtime‚2091DzÂZ‹Ô /loss/FRt4/assignment_loss‚4.084052336459258 2loss/SRqqt2/assignment_loss‚1.7938232317164138 2loss/SRqqt3/assignment_loss‚1.2652181169744265 0loss/FBt3/assignment_loss‚0.3198158795005051 0loss/FBt4/assignment_loss‚0.4800982028245926 /loss/FRt2/detection_loss‚2.3014356134568943 1loss/SRqqt4/detection_loss‚2.0355821345366683 % _timestamp‚1.7884469757536583e+09 2loss/SRqqt4/assignment_loss‚1.9528676297734766 1loss/SRqqt1/detection_loss‚2.7702369581569326 _step‚213 epoch‚2 2loss/SRqqt1/assignment_loss‚2.2572408282395564 0loss/FBt2/assignment_loss‚0.5683764825225353 1loss/SRqqt2/detection_loss‚2.4314727493234582 /loss/FRt3/assignment_loss‚2.567493358525363 /loss/FRt2/assignment_loss‚4.480601289693047 /loss/FRt4/detection_loss‚2.3878427774322275 /loss/FBt3/detection_loss‚0.8939738588793794 trainer/global_step‚2599 /loss/FRt1/assignment_loss‚4.743962095629785 0loss/FBt1/assignment_loss‚0.5678928752968204 1loss/SRqqt3/detection_loss‚1.5359143407668097 .loss/FBt2/detection_loss‚2.327839026848475 &loss/total_loss‚55.009135935991495 /loss/FRt1/detection_loss‚2.8725506644095145 /loss/FRt3/detection_loss‚1.8808261383663525 /loss/FBt1/detection_loss‚3.0979170753405643 .loss/FBt4/detection_loss‚5.392102271318436 _runtime‚2091.225195067àE'S_ŒÙ /loss/FRt2/assignment_loss‚4.480601289693047 /loss/FRt4/detection_loss‚2.3878427774322275 /loss/FBt3/detection_loss‚0.8939738588793794 trainer/global_step‚2599 /loss/FRt1/assignment_loss‚4.743962095629785 0loss/FBt1/assignment_loss‚0.5678928752968204 1loss/SRqqt3/detection_loss‚1.5359143407668097 .loss/FBt2/detection_loss‚2.327839026848475 &loss/total_loss‚55.009135935991495 /loss/FRt1/detection_loss‚2.8725506644095145 /loss/FRt3/detection_loss‚1.8808261383663525 /loss/FBt1/detection_loss‚3.0979170753405643 .loss/FBt4/detection_loss‚5.392102271318436 _runtime‚2091.225195067 /loss/FRt4/assignment_loss‚4.084052336459258 2loss/SRqqt2/assignment_loss‚1.7938232317164138 2loss/SRqqt3/assignment_loss‚1.2652181169744265 0loss/FBt3/assignment_loss‚0.3198158795005051 0loss/FBt4/assignment_loss‚0.4800982028245926 /loss/FRt2/detection_loss‚2.3014356134568943 1loss/SRqqt4/detection_loss‚2.0355821345366683 % _timestamp‚1.7884469757536583e+09 2loss/SRqqt4/assignment_loss‚1.9528676297734766 1loss/SRqqt1/detection_loss‚2.7702369581569326 _step‚213 epoch‚2 2loss/SRqqt1/assignment_loss‚2.2572408282395564 0loss/FBt2/assignment_loss‚0.5683764825225353 1loss/SRqqt2/detection_loss‚2.4314727493234582 /loss/FRt3/assignment_loss‚2.567493358525363Õw­i“‚(:ˆ ˆ’æÔäŸÄþ gpu.0.gpu‚45.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.482' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8964 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0wJ9ØMŽ‚(: ˆ’æÔÛÔÿ disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 79.08984375 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 79.08984375 disk.sda3.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0" proc.memory.rssMB‚ 2642.7109375$ disk./.usageGB‚298.7873306274414 disk.sda.out‚ 79.08984375 disk.drbd1716.in‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚8318.71630859375 proc.cpu.threads‚17% disk.nvme0n1.out‚8318.71630859375 network.sent‚ 5.285864e+06 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2080.out‚0 cpu‚25.353697345243386 disk.sda.in‚0 disk.sda1.out‚0 disk.dm-1.in‚0 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚8318.71630859375 disk.drbd2266.in‚0$ memory_percent‚97.10814952850342 network.recv‚778955) disk./.usagePercent‚20.05186845767626 disk.rbd1.out‚ 10.896484375 disk.drbd2265.in‚0( proc.memory.availableMB‚ 473.80078125 disk.rbd1.in‚0 disk.drbd2080.in‚0) proc.memory.percent‚16.12982749938965‚6´ ž‚(:“ —’æÔíüŽý gpu.0.gpu‚32.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚126.371' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.274199999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¼ÓPF‚(:» —’æÔç’‰€ disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.sda3.out‚ 79.578125$ memory_percent‚97.10836410522461 network.recv‚783866 disk.rbd0.in‚0 disk.rbd1.out‚ 11.107421875 disk.dm-1.in‚0 disk.drbd1716.out‚0 cpu‚25.351563025028614 network.sent‚ 5.393695e+06 disk.drbd2266.out‚0 disk.drbd2265.in‚0" proc.memory.rssMB‚ 2642.7109375 proc.cpu.threads‚17" disk.dm-0.out‚8378.02978515625 disk.sda2.in‚0 disk.dm-1.out‚ 79.578125 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.sda2.out‚0 disk.nvme0c0n1.in‚0.078125) proc.memory.percent‚16.12982749938965 disk.sda1.in‚0% disk.nvme0n1.out‚8378.02978515625* disk./.usagePercent‚20.053848931961145 disk.drbd2266.in‚0 disk.drbd1716.in‚0 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚8378.02978515625 disk.sda3.in‚0& proc.memory.availableMB‚ 473.765625 disk.rbd0.out‚0 disk.sda.out‚ 79.578125 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0$ disk./.usageGB‚298.8168411254883>øìý2‘- _wandbruntime‚2127  _runtime‚2127Nâµ­É’Ã _step‚214 ' lr-AdamW/pg1‚0.00012128947152085744 ' lr-AdamW/pg2‚0.00012128947152085744 trainer/global_step‚2649 % _timestamp‚1.7884470118112295e+09 _runtime‚2127.282829134³Dx˜Î“È ' lr-AdamW/pg1‚0.00012128947152085744 ' lr-AdamW/pg2‚0.00012128947152085744 trainer/global_step‚2649 % _timestamp‚1.7884470118112295e+09 _runtime‚2127.282829134 _step‚214Ö¢aG"2”- _wandbruntime‚2127  _runtime‚2127oK“Þ•ÓO E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12750208 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50019135 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚156211067792896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚153993031058944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50019135 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179175936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚156211742157824 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚168167428602368 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚149958643067904 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49999098 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2218036733952 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2218693095936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚88530689 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚170496349263360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚156211742157824 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚153993049061888 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚156211067792896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚153993049061888 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚149958660107264 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsM…FñXonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22470404 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35220614 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2210524334893 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚153993031058944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚152168512965045 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚152169184442157 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38511552 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22470406 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2218036733952 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚170496528439296 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2329075896832 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50019135 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚156211067792896 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚153993031058944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50019135 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179175936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚156211742157824 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚168167428602368 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚149958643067904 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49999098 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2218036733952 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2218693095936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚88530689 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚170496349263360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚156211742157824 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚153993049061888 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚156211067792896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚153993049061888 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚149958660107264 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22470404 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35220614 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2210524334893 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚153993031058944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚152168512965045 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚152169184442157×f{þ„2—- _wandbruntime‚2128  _runtime‚2128ÆÖ0câ&˜ÜM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚156268914637824 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2329940615168 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88563185 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚168229079340032 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚2649 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88543120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚150013564237824 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚154049402100736 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚154049384097792 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2329770712576 _runtime‚2128.218282624 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚154049384097792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35232809 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12754237 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚150013547198464 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚156268240259584 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚152224905007889 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50017552 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚170559043895296 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38525568 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22478574 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚152224233530569 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50037615 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12754670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚170558850052608 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2211340770065 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38525570 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35233244 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2219512537088 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _step‚216 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2219512537088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2218856161792 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚156268914637824 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 $ _timestamp‚1.788447012746087e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50017552 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88563185 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚88543120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22478572 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚154049402100736 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38525568 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2210686332105 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0Ñ­¡“!™áM MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _step‚216 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2219512537088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2218856161792 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚156268914637824 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 $ _timestamp‚1.788447012746087e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50017552 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88563185 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚88543120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22478572 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚154049402100736 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38525568 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2210686332105 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚156268914637824 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2329940615168 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88563185 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚168229079340032 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚2649 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88543120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚150013564237824 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚154049402100736 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚154049384097792 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2329770712576 _runtime‚2128.218282624 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚154049384097792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35232809 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12754237 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚150013547198464 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚156268240259584 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚152224905007889 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50017552 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚170559043895296 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38525568 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end&„r¨T/segment.small_pool.current‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22478574 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚152224233530569 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50037615 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12754670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚170558850052608 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2211340770065 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38525570 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35233244 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2219512537088 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0Øä²ú2š- _wandbruntime‚2128  _runtime‚2128ã\{T›Î 1loss/SRqqt3/detection_loss‚1.3950855541920317 .loss/FBt4/detection_loss‚5.718694284558296 2loss/SRqqt3/assignment_loss‚1.5633644096661306 1loss/SRqqt2/detection_loss‚1.7502735835313796 .loss/FBt1/detection_loss‚3.058644317663633 .loss/FBt2/detection_loss‚2.554857629537582 1loss/SRqqt1/assignment_loss‚2.715852244032754 /loss/FRt1/assignment_loss‚5.480185491067392 /loss/FBt1/assignment_loss‚0.758936687156515 /loss/FRt1/detection_loss‚3.0332911014556885 /loss/FRt2/detection_loss‚2.3866826029200303 0loss/SRqqt4/detection_loss‚1.668784058187157 &loss/total_loss‚54.423186519165185 _step‚217 /loss/FRt2/assignment_loss‚4.103214539979634 0loss/FRt3/assignment_loss‚2.3448743288931637 2loss/SRqqt2/assignment_loss‚1.9585867369174956 0loss/FBt3/assignment_loss‚0.2668015722029065 0loss/SRqqt1/detection_loss‚2.783271250901399 epoch‚2 % _timestamp‚1.7884470127478278e+09 .loss/FRt3/detection_loss‚2.178094809955877 trainer/global_step‚2649 _runtime‚2128.219338266 0loss/FRt4/assignment_loss‚3.1992606229252285 2loss/SRqqt4/assignment_loss‚1.9050807049497962 0loss/FBt2/assignment_loss‚0.2793798020513231 0loss/FBt4/assignment_loss‚0.2224431447684765 .loss/FRt4/detection_loss‚2.324372039238612 .loss/FBt3/detection_loss‚0.773155002412681ø®¬åYœÓ 0loss/FRt4/assignment_loss‚3.1992606229252285 2loss/SRqqt4/assignment_loss‚1.9050807049497962 0loss/FBt2/assignment_loss‚0.2793798020513231 0loss/FBt4/assignment_loss‚0.2224431447684765 .loss/FRt4/detection_loss‚2.324372039238612 .loss/FBt3/detection_loss‚0.773155002412681 1loss/SRqqt3/detection_loss‚1.3950855541920317 .loss/FBt4/detection_loss‚5.718694284558296 2loss/SRqqt3/assignment_loss‚1.5633644096661306 1loss/SRqqt2/detection_loss‚1.7502735835313796 .loss/FBt1/detection_loss‚3.058644317663633 .loss/FBt2/detection_loss‚2.554857629537582 1loss/SRqqt1/assignment_loss‚2.715852244032754 /loss/FRt1/assignment_loss‚5.480185491067392 /loss/FBt1/assignment_loss‚0.758936687156515 /loss/FRt1/detection_loss‚3.0332911014556885 /loss/FRt2/detection_loss‚2.3866826029200303 0loss/SRqqt4/detection_loss‚1.668784058187157 &loss/total_loss‚54.423186519165185 _step‚217 /loss/FRt2/assignment_loss‚4.103214539979634 0loss/FRt3/assignment_loss‚2.3448743288931637 2loss/SRqqt2/assignment_loss‚1.9585867369174956 0loss/FBt3/assignment_loss‚0.2668015722029065 0loss/SRqqt1/detection_loss‚2.783271250901399 epoch‚2 % _timestamp‚1.7884470127478278e+09 .loss/FRt3/detection_loss‚2.178094809955877 trainer/global_step‚2649 _runtime‚2128.219338266Ùâáhž‚(:“ ¦’æÔ§™»ý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚145.402' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.080399999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¾ÑˆMž‚(: ¦’æÔÅ·…€ disk.drbd2265.in‚0 disk.sda2.in‚0) proc.memory.percent‚16.12982749938965 disk.rbd1.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda2.out‚0 network.recv‚789037 disk.drbd2265.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚8426.86572265625 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 proc.cpu.threads‚17) disk./.usagePercent‚20.05523726081933 disk.dm-1.out‚ 80.18359375 disk.drbd2080.out‚0" disk.dm-0.out‚8426.86572265625 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚8426.86572265625 disk.sda3.out‚ 80.18359375$ memory_percent‚97.11089134216309' proc.memory.availableMB‚ 473.3515625% disk./.usageGB‚298.83752822875977 disk.rbd1.out‚ 11.169921875 disk.dm-1.in‚0 cpu‚25.349468305191518 network.sent‚ 5.400397e+06 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.sda.out‚ 80.16796875" proc.memory.rssMB‚ 2642.7109375ºû—“Ÿ‚(:ˆ µ’æÔ¸³Éý gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.123' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8246 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0à’…ûM ‚(: µ’æÔó·Óÿ disk.rbd0.in‚0* disk./.usagePercent‚20.056343725382316 disk.dm-0.in‚0.078125 disk.sda.out‚ 80.72265625 disk.dm-1.out‚ 80.72265625 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.rbd0.out‚0 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚8474.86279296875 disk.sda3.out‚ 80.72265625$ disk./.usageGB‚298.8540153503418 disk.drbd2080.out‚0 disk.sda3.in‚0" disk.dm-0.out‚8474.86279296875 disk.sda.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 11.169921875 disk.drbd2266.in‚0$ memory_percent‚97.10938930511475 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2266.out‚0( proc.memory.availableMB‚ 473.59765625 network.sent‚ 5.508126e+06 network.recv‚794339" proc.memory.rssMB‚ 2642.7109375 disk.dm-1.in‚0) proc.memory.percent‚16.12982749938965 cpu‚25.34741178318693 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚8474.86279296875 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚07'†“¡‚(:ˆ Ä’æÔƒý gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.841' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.1682 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0>dÔE¢‚(:º Ä’æÔŠê ÿ# disk./.usageGB‚298.883487701416' disk.nvme0c0n1.out‚8518.92919921875 disk.dm-1.out‚ 81.1953125 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.dm-1.in‚0 network.sent‚ 5.514566e+06 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda.out‚ 81.1953125% disk.nvme0n1.out‚8518.92919921875) proc.memory.percent‚16.12982749938965 disk.drbd2266.in‚0" disk.dm-0.out‚8518.92919921875 disk.sda.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.sda3.out‚ 81.1953125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2642.7109375 cpu‚25.34538354181877 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0# memory_percent‚97.1109390258789* disk./.usagePercent‚20.058321639591856 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.rbd1.out‚ 11.498046875% proc.memory.availableMB‚ 473.34375 network.recv‚799476 disk.drbd2080.out‚0 disk.sda1.out‚0P© 2£- _wandbruntime‚2164  _runtime‚2164Œ/'ÿɤà _step‚218 ' lr-AdamW/pg1‚0.00012091412238669505 ' lr-AdamW/pg2‚0.00012091412238669505 trainer/global_step‚2699 % _timestamp‚1.7884470488037384e+09 _runtime‚2164.275327379ƒOÝCÎ¥È ' lr-AdamW/pg1‚0.00012091412238669505 ' lr-AdamW/pg2‚0.00012091412238669505 trainer/global_step‚2699 % _timestamp‚1.7884470488037384e+09 _runtime‚2164.275327379 _step‚218Ú̆Õ2¦- _wandbruntime‚2164  _runtime‚2164°"7JÙ'§ÓO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35854439 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39212352 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚156810683001344 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39212354 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚422 _runtime‚2164.276632542 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚156810683001344 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚154954554832453 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚159069705541120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50945187 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚159070379906048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185467392 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚152703849595904 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2259022539776 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22878806 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7884470488045743e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50925150 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚173621759115264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 trainer/global_step‚2699 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚159070379906048 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚152703866635264 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚156810701004288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50925150 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2259022539776 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚90157541 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚171249965485568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2371793629696 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50945187 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12975635 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39212354̳Þ'¨ØO XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39212352 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚156810683001344 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39212354 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚422 _runtime‚2164.276632542 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚156810683001344 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚154954554832453 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚159069705541120 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50945187 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚159070379906048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185467392 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚152703849595904 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2259022539776 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22878806 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7884470488045743e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50925150 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚173621759115264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 trainer/global_step‚2699 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚159070379906048 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚152703866635264 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚156810701004288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50925150 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2259022539776 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚90157541 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚171249965485568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2371793629696 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50945187 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12975635 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39212354 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35854439 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944Û)*n2©- _wandbruntime‚2164  _runtime‚2164o“¸±È ªÜM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90190037 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39226370 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39226368 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2259841967616 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39226368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2260498342912 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35867094 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 % _timestamp‚1.7884470495124137e+09 _runtime‚ 2164.98487141 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚156867036040192 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚444 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚171311616223232 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚152758770765824 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2252176109473 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚90169972 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50963667 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚156867036040192 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2260498342912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 6743„Œû@78240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚159126878007808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚152758753726464 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90190037 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚173684451751424 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚446 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50963667 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90169972 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚156867054043136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22886974 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚171311640163328 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2251521671513 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35867540 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚156867054043136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22886972 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2372811588096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2372641685504 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12980122 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90169972 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚156867054043136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22886974 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚171311640163328 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2251521671513 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35867540 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚156867054043136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22886972 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2372811588096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2372641685504 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12980122 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39226370 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39226368 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2259841967616 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39226368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2260498342912 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35867094 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 % _timestamp‚1.7884470495124137e+09 _runtime‚ 2164.98487141 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚156867036040192 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚444 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚171311616223232 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚152758770765824 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2252176109473 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚90169972 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50963667 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚156867036040192 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2260498342912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚159126878007808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚152758753726464 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90190037 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚173684451751424 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚446 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0ܵ³$±2¬- _wandbruntime‚2164  _runtime‚2164P¦ZÚY­Ó /loss/FRt3/detection_loss‚1.7330147022740883 .loss/FBt2/detection_loss‚2.956819095394828 %loss/total_loss‚55.91861820208422 .loss/FBt4/detection_loss‚6.263276696205139 /loss/FRt2/assignment_loss‚4.096327957718871 0loss/SRqqt1/detection_loss‚2.636750172346066 2loss/SRqqt2/assignment_loss‚1.8594502461227504 epoch‚2 % _timestamp‚1.7884470495147254e+09 2loss/SRqqt1/assignment_loss‚2.3800466626118393 1loss/SRqqt3/assignment_loss‚1.635161072597839 0loss/SRqqt2/detection_loss‚2.012396517125043 0loss/FRt3/assignment_loss‚2.1238424568845513 0loss/FBt1/assignment_loss‚0.3330485729669179 1loss/FBt4/assignment_loss‚0.18307175735632578 1loss/SRqqt3/detection_loss‚1.5114888872485608 .loss/FRt2/detection_loss‚2.373124494109043 /loss/FRt4/assignment_loss‚4.223473246097565 /loss/FRt1/detection_loss‚2.8350336477160454 1loss/SRqqt4/detection_loss‚1.7705602154868547 /loss/FBt3/detection_loss‚0.8627461888439333 trainer/global_step‚2699 _step‚221 1loss/FBt3/assignment_loss‚0.28410224097154985 /loss/FRt4/detection_loss‚2.3969163900613784 .loss/FBt1/detection_loss‚4.061619944042629 _runtime‚2164.986247357 0loss/FRt1/assignment_loss‚5.0454820320010185 2loss/SRqqt4/assignment_loss‚1.8662418107517431 1loss/FBt2/assignment_loss‚0.47462319514964046#'g^®Ø /loss/FRt2/assignment_loss‚4.096327957718871 0loss/SRqqt1/detection_loss‚2.636750172346066 2loss/SRqqt2/assignment_loss‚1.8594502461227504 epoch‚2 % _timestamp‚1.7884470495147254e+09 2loss/SRqqt1/assignment_loss‚2.3800466626118393 1loss/SRqqt3/assignment_loss‚1.635161072597839 0loss/SRqqt2/detection_loss‚2.012396517125043 0loss/FRt3/assignment_loss‚2.1238424568845513 0loss/FBt1/assignment_loss‚0.3330485729669179 1loss/FBt4/assignment_loss‚0.18307175735632578 1loss/SRqqt3/detection_loss‚1.5114888872485608 .loss/FRt2/detection_loss‚2.373124494109043 /loss/FRt4/assignment_loss‚4.223473246097565 /loss/FRt1/detection_loss‚2.8350336477160454 1loss/SRqqt4/detection_loss‚1.7705602154868547 /loss/FBt3/detection_loss‚0.8627461888439333 trainer/global_step‚2699 _step‚221 1loss/FBt3/assignment_loss‚0.28410224097154985 /loss/FRt4/detection_loss‚2.3969163900613784 .loss/FBt1/detection_loss‚4.061619944042629 _runtime‚2164.986247357 0loss/FRt1/assignment_loss‚5.0454820320010185 2loss/SRqqt4/assignment_loss‚1.8662418107517431 1loss/FBt2/assignment_loss‚0.47462319514964046 /loss/FRt3/detection_loss‚1.7330147022740883 .loss/FBt2/detection_loss‚2.956819095394828 %loss/total_loss‚55.91861820208422 .loss/FBt4/detection_loss‚6.263276696205139ÝÆ4²“¯‚(:ˆ Ó’æÔœŠ†ý gpu.0.gpu‚46.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.342' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0684 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0UŒñhM°‚(: ӒæÔží“ÿ disk.drbd2266.in‚0 network.sent‚ 5.51967e+06* disk./.usagePercent‚20.058712051081574 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.rbd0.in‚0 disk.sda.out‚ 81.80078125 disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0' disk.nvme0c0n1.out‚8575.04736328125 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.dm-1.out‚ 81.80078125 network.recv‚802947 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚0 disk.drbd2265.in‚0$ disk./.usageGB‚298.8893051147461 disk.rbd1.out‚ 11.623046875 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚8575.04736328125 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.sda3.out‚ 81.80078125" disk.dm-0.out‚8575.04736328125 disk.sda.in‚0$ memory_percent‚97.09594249725342( proc.memory.availableMB‚ 475.80078125" proc.memory.rssMB‚ 2642.7109375 proc.cpu.threads‚17) proc.memory.percent‚16.12982749938965 cpu‚25.343375833830738¦a…ž±‚(:“ â’æԛϺþ gpu.0.gpu‚41.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚142.602' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.520400000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ëM{C²‚(:¸ â’æÔ™±ˆ€ disk.rbd0.in‚0 disk.sda.in‚0 disk.sda2.out‚0 disk.dm-1.out‚82.53125' disk.nvme0c0n1.out‚8631.35205078125 disk.rbd0.out‚0$ memory_percent‚97.09746837615967( proc.memory.availableMB‚ 475.55078125) proc.memory.percent‚16.12982749938965 disk.drbd2080.out‚0 network.sent‚ 5.6289e+06 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125" proc.memory.rssMB‚ 2642.7109375 disk.sda.out‚82.53125" disk.dm-0.out‚8631.35205078125 disk.sda2.in‚0 disk.drbd2080.in‚0 cpu‚25.34137156834363 proc.cpu.threads‚17 disk.drbd2265.out‚0* disk./.usagePercent‚20.060571945817827 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 11.685546875 disk.nvme0n1.in‚0.078125 network.recv‚809911% disk.nvme0n1.out‚8631.35205078125 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.sda3.in‚0 disk.sda3.out‚82.53125% disk./.usageGB‚298.91701889038086 disk.drbd1716.in‚0äV¯R2³- _wandbruntime‚2201  _runtime‚2201HñîãÉ´Ã _step‚222 ' lr-AdamW/pg1‚0.00012052677548490224 ' lr-AdamW/pg2‚0.00012052677548490224 trainer/global_step‚2749 % _timestamp‚1.7884470861270971e+09 _runtime‚2201.598671158«dëÎµÈ ' lr-AdamW/pg1‚0.00012052677548490224 ' lr-AdamW/pg2‚0.00012052677548490224 trainer/global_step‚2749 % _timestamp‚1.7884470861270971e+09 _runtime‚2201.598671158 _step‚222ÞxÏ2¶- _wandbruntime‚2201  _runtime‚2201ès~ø$·ÒO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91784393 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 trainer/global_step‚2749 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2414666491392 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚155449073163264 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2292195013709 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚174332502368768 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39913152 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2300008345600 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23287204 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176207360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51851202 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13202231 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39913154 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2300664707584 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 $ _timestamp‚1.788447086127788e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚161928343289344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2300664707584 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚161928343289344 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36489435 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23287206 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚157740596699861 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51871239 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2300008345600 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51851202 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚174332526308864 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚159628352946688 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚176747168860160 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚176747369007616 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚161929017654272 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚159628334943744 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚91764354 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚159628334943744 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚159628352946688 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39913152 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36489860 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚425 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚423 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚155449056123904 _runtime‚2201.599564163 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51871239 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200147456 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13202654 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2300008345600 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23287204 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176207360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51851202 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13202231 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39913154 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2300664707584 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 $ _timestamp‚1.788447086127788e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚161928343289344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2300664707584 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚161928343289344 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36489435 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23287206 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚157740596699861 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51871239 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2300008345600 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51851202 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚174332526308864 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚159628352946688 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚176747168860160 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚176747369007616 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚161929017654272 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚159628334943744 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚91764354 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚159628334943744 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚159628352946688 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚39913152 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36489860 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚425 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚423 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚155449056123904 _runtime‚2201.599564163 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51871239 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200147456 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13202654 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51889719 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚176809866886656 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36502136 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39927168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13207203 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚161986190134272 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39927170 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2292357010921 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚157796988742705 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23295372 trainer/global_step‚2749 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91816889 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2301484148736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2300827773440 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚176809677238272 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚174394177046528 _runtime‚2202.289187299 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51889719 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13206764 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91816889 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚174394153106432 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51869656 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189648384 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚159684705985536 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚159684687982592 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36502577 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚161985515756032 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91796824 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚155503960254464 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7884470868172302e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39927168 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51869656 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2415524131840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2293011448881 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚161986190134272 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165708288 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944¡¢¹¦è&»âM \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚176809866886656 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36502136 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39927168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13207203 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚161986190134272 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39927170 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2292357010921 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚157796988742705 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23295372 trainer/global_step‚2749 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91816889 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2301484148736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2300827773440 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚176809677238272 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚174394177046528 _runtime‚2202.289187299 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51889719 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13206764 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91816889 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚174394153106432 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚51869656 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189648384 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚159684705985536 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚159684687982592 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36502577 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚161985515756032 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91796824 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚155503960254464 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7884470868172302e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39927168 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51869656 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2415524131840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2293011448881 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚161986190134272 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165708288 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2301484148736 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51889719 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2àqæ‚ß2¼- _wandbruntime‚2202  _runtime‚2202q*žY½Ó _step‚225 1loss/FBt3/assignment_loss‚0.29174080919772577 /loss/FRt4/detection_loss‚2.6115329372031346 .loss/FBt1/detection_loss‚2.671236291527748 1loss/SRqqt4/detection_loss‚1.9051776243406429 /loss/FBt2/detection_loss‚2.5288047313690187 /loss/FBt3/detection_loss‚0.8161228520435874 /loss/FBt4/detection_loss‚3.5145756701628366 trainer/global_step‚2749 0loss/FRt2/assignment_loss‚3.6261726827234835 /loss/FBt1/assignment_loss‚0.910489899546306 0loss/FBt4/assignment_loss‚0.6799649881816853 /loss/FRt4/assignment_loss‚3.504240666116987 1loss/SRqqt4/assignment_loss‚1.562055880824725 .loss/FRt1/detection_loss‚2.947708668247346 /loss/FRt3/detection_loss‚1.9339623432606459 1loss/SRqqt1/detection_loss‚2.5883551176750297 % _timestamp‚1.7884470868190362e+09 1loss/SRqqt3/assignment_loss‚1.820756467489096 epoch‚2 1loss/SRqqt2/assignment_loss‚2.221034428587666 1loss/SRqqt3/detection_loss‚1.2582208532553452 0loss/FRt1/assignment_loss‚4.8590331692849436 0loss/FRt3/assignment_loss‚2.3773634930451712 2loss/SRqqt1/assignment_loss‚2.4013390902316933 .loss/FRt2/detection_loss‚2.385658375314764 0loss/SRqqt2/detection_loss‚1.875108469967489 _runtime‚2202.290505767 1loss/FBt2/assignment_loss‚0.46436922252090085 &loss/total_loss‚51.755024732117974’Êk ƒ¾Ø 1loss/SRqqt3/detection_loss‚1.2582208532553452 0loss/FRt1/assignment_loss‚4.8590331692849436 0loss/FRt3/assignment_loss‚2.3773634930451712 2loss/SRqqt1/assignment_loss‚2.4013390902316933 .loss/FRt2/detection_loss‚2.385658375314764 0loss/SRqqt2/detection_loss‚1.875108469967489 _runtime‚2202.290505767 1loss/FBt2/assignment_loss‚0.46436922252090085æò6,Û &loss/total_loss‚51.755024732117974 _step‚225 1loss/FBt3/assignment_loss‚0.29174080919772577 /loss/FRt4/detection_loss‚2.6115329372031346 .loss/FBt1/detection_loss‚2.671236291527748 1loss/SRqqt4/detection_loss‚1.9051776243406429 /loss/FBt2/detection_loss‚2.5288047313690187 /loss/FBt3/detection_loss‚0.8161228520435874 /loss/FBt4/detection_loss‚3.5145756701628366 trainer/global_step‚2749 0loss/FRt2/assignment_loss‚3.6261726827234835 /loss/FBt1/assignment_loss‚0.910489899546306 0loss/FBt4/assignment_loss‚0.6799649881816853 /loss/FRt4/assignment_loss‚3.504240666116987 1loss/SRqqt4/assignment_loss‚1.562055880824725 .loss/FRt1/detection_loss‚2.947708668247346 /loss/FRt3/detection_loss‚1.9339623432606459 1loss/SRqqt1/detection_loss‚2.5883551176750297 % _timestamp‚1.7884470868190362e+09 1loss/SRqqt3/assignment_loss‚1.820756467489096 epoch‚2 1loss/SRqqt2/assignment_loss‚2.221034428587666á± º‡“¿‚(:ˆ ñ’æÔƒ°óü gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.699' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.3398 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 .8À‚(:­ ñ’æÔ§€ÿ disk.drbd1716.out‚0 disk.sda3.out‚83.1875( proc.memory.availableMB‚ 475.73046875 network.recv‚812899 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0) proc.memory.percent‚16.12982749938965$ disk./.usageGB‚298.9174156188965 disk.drbd2080.out‚0% disk.nvme0c0n1.out‚8690.525390625 disk.sda3.in‚0" proc.memory.rssMB‚ 2642.7109375 disk.drbd2266.out‚0 disk.sda.in‚0 disk.rbd1.out‚ 11.685546875 disk.dm-1.out‚83.1875 disk.drbd2080.in‚0# memory_percent‚97.0963716506958 disk.drbd1716.in‚0 disk.drbd2265.out‚0 proc.cpu.threads‚17 network.sent‚ 5.634104e+06 disk.dm-0.out‚8690.525390625 disk.sda1.out‚0# disk.nvme0n1.out‚8690.525390625 disk.dm-0.in‚0.078125 disk.nvme0c0n1.in‚0.078125) disk./.usagePercent‚20.06059857060139 disk.rbd0.out‚0 cpu‚25.3394404926702 disk.sda.out‚83.1875 disk.sda2.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0̱ƒº“Á‚(:ˆ €“æÔÈ©”ý gpu.0.gpu‚47.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚168.219' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.6438 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‹±bG‚(:¼ €“æÔŸ ˜ÿ" proc.memory.rssMB‚ 2642.7109375$ disk./.usageGB‚298.9317817687988 disk.dm-1.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.sda3.out‚ 83.77734375 disk.sda1.out‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125& proc.memory.availableMB‚ 475.234375 cpu‚25.337394345530754 network.sent‚ 5.743005e+06 disk.drbd2266.in‚0 disk.dm-0.out‚8746.177734375 disk.sda2.in‚0 disk.rbd1.out‚ 11.7177734375) proc.memory.percent‚16.12982749938965 disk.dm-1.out‚ 83.77734375 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125# disk.nvme0n1.out‚8746.177734375$ memory_percent‚97.09939956665039* disk./.usagePercent‚20.061562694975336 disk.drbd1716.in‚0 proc.cpu.threads‚17 disk.sda.out‚ 83.77734375 disk.sda1.in‚0 network.recv‚819551 disk.drbd2265.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0% disk.nvme0c0n1.out‚8746.177734375 disk.rbd0.in‚0…0DޓÂ(:ˆ “æÔиðü gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.699' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.3398 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0+Ô™–BÄ‚(:· “æÔÕ¹€ÿ disk.sda2.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0) proc.memory.percent‚16.12982749938965 cpu‚25.335494749193263 disk.sda.out‚84.28125 disk.sda1.in‚0$ disk.nvme0n1.out‚8802.9658203125 disk.drbd1716.in‚0" proc.memory.rssMB‚ 2642.7109375 disk.dm-1.out‚84.28125 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0& disk.nvme0c0n1.out‚8802.9658203125 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125! disk.dm-0.out‚8802.9658203125 network.sent‚ 5.748193e+06 disk.drbd2080.out‚0 disk.sda3.out‚84.28125$ memory_percent‚97.09641933441162 disk.rbd1.out‚ 11.9287109375 disk.dm-1.in‚0 disk.drbd2266.out‚0( proc.memory.availableMB‚ 475.72265625 network.recv‚822839 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2266.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚298.9306335449219 disk.sda.in‚0 disk.sda1.out‚0) disk./.usagePercent‚20.06148563670753%k2Å- _wandbruntime‚2238  _runtime‚2238ožKwÇÆÁ trainer/global_step‚2799 $ _timestamp‚1.788447122841255e+09 _runtime‚ 2238.31284843 _step‚226 ' lr-AdamW/pg1‚0.00012012751153799277 ' lr-AdamW/pg2‚0.00012012751153799277aÛ¹§ÌÇÆ $ _timestamp‚1.788447122841255e+09 _runtime‚ 2238.31284843 _step‚226 ' lr-AdamW/pg1‚0.00012012751153799277 ' lr-AdamW/pg2‚0.00012012751153799277 trainer/global_step‚2799âÑ_2È- _wandbruntime‚2238  _runtime‚2238éTÙ'ÉÓO XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52797291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚164787655402496 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚162445986886144 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40613952 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52777254 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2341650513408 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚164786981037568 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2341650513408 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13430136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚160527310044381 trainer/global_step‚2799 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2340994151424 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚177415063192064 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13429693 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2457755935232 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40613952 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚158194279691264 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2332375915365 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93411245 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2333030353117 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚164786981037568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40613954 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚162446004889088“ÛÉÞ'ÊØO LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚2238.313792833 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23695606 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚164786981037568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40613954 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚162446004889088 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52797291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚164787655402496 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚162445986886144 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40613952 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚52777254 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2341650513408 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚164786981037568 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2341650513408 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13430136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚160527310044381 trainer/global_step‚2799 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2340994151424 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚177415063192064 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13429693 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2457755935232 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40613952 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚158194279691264 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2332375915365 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93411245 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2333030353117 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37137907 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23703774 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13434135 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93443741 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚179935134786048 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚162502357927936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚451 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551232 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2458608407552 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2341813579264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52795708 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚162502339924992 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚160583030610113 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚158249166782464 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2341813579264 SEDeviceStatsMonitor.on_train_batch_end/requesÔd«Mted_bytes.small_pool.peak‚ 861853517 trainer/global_step‚2799 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93423676 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52815771 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93443741 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚164844827882496 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚228 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37138360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚160582359132793 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚ 2238.99428434 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚164844153504256 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40627970 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚162502357927936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2342469954560 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40627970 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2333192350329 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52815771 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93423676 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚158249183821824 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944N-‚Áæ&ÍàM _step‚228 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37138360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚160582359132793 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚ 2238.99428434 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚164844153504256 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40627970 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚162502357927936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2342469954560 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40627970 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2333192350329 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52815771 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93423676 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚158249183821824 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37137907 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23703774 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13434135 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93443741 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚179935134786048 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚162502357927936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚451 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187551232 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2458608407552 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2341813579264 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52795708 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚162502339924992 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚160583030610113 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚158249166782464 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2341813579264 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 trainer/global_step‚2799 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚93423676 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52815771 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93443741 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚164844827882496 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928äç?m€2Î- _wandbruntime‚2238  _runtime‚2238[}UÏÏ 1loss/FBt4/assignment_loss‚0.34303830564022064 /loss/FRt3/detection_loss‚1.9279403827552284 _runtime‚2238.995400006 1loss/SRqqt4/assignment_loss‚1.966906348141757 /loss/FRt2/detection_loss‚2.4446764829310963 1loss/SRqqt4/detection_loss‚1.9348344531926243 %loss/total_loss‚55.77927836627954 /loss/FRt1/assignment_loss‚5.038875389099121 1loss/SRqqt2/assignment_loss‚2.045924447077726 trainer/global_step‚2799 % _timestamp‚1.7884471235239015e+09 _step‚229 2loss/SRqqt3/assignment_loss‚1.8463999491930008 0loss/FBt1/assignment_loss‚0.8332618088469543 .loss/FRt1/detection_loss‚3.065497672557831 1loss/SRqqt1/detection_loss‚2.9609533150990806 .loss/FBt1/detection_loss‚3.385986804962158 .loss/FBt4/detection_loss‚5.765496989091237 0loss/FRt4/assignment_loss‚3.7432156801223755 0loss/FBt2/assignment_loss‚0.4267041515388428 epoch‚2 /loss/FRt2/assignment_loss‚4.176194974716673 0loss/SRqqt1/assignment_loss‚2.20270761957875 /loss/FBt3/detection_loss‚0.8099908401800578 /loss/FRt3/assignment_loss‚2.400844546301024 1loss/FBt3/assignment_loss‚0.12480183700459196 .loss/FRt4/detection_loss‚2.734468574033064 1loss/SRqqt2/detection_loss‚1.8567746898957662 0loss/SRqqt3/detection_loss‚1.301630293726921 /loss/FBt2/detection_loss‚2.4421528105934462…¿™ÍZÐÔ 0loss/FBt2/assignment_loss‚0.4267041515388428 epoch‚2 /loss/FRt2/assignment_loss‚4.176194974716673 0loss/SRqqt1/assignment_loss‚2.20270761957875 /loss/FBt3/detection_loss‚0.8099908401800578 /loss/FRt3/assignment_loss‚2.400844546301024 1loss/FBt3/assignment_loss‚0.12480183700459196 .loss/FRt4/detection_loss‚2.734468574033064 1loss/SRqqt2/detection_loss‚1.8567746898957662 0loss/SRqqt3/detection_loss‚1.301630293726921 /loss/FBt2/detection_loss‚2.4421528105934462 1loss/FBt4/assignment_loss‚0.34303830564022064 /loss/FRt3/detection_loss‚1.9279403827552284 _runtime‚2238.995400006 1loss/SRqqt4/assignment_loss‚1.966906348141757 /loss/FRt2/detection_loss‚2.4446764829310963 1loss/SRqqt4/detection_loss‚1.9348344531926243 %loss/total_loss‚55.77927836627954 /loss/FRt1/assignment_loss‚5.038875389099121 1loss/SRqqt2/assignment_loss‚2.045924447077726 trainer/global_step‚2799 % _timestamp‚1.7884471235239015e+09 _step‚229 2loss/SRqqt3/assignment_loss‚1.8463999491930008 0loss/FBt1/assignment_loss‚0.8332618088469543 .loss/FRt1/detection_loss‚3.065497672557831 1loss/SRqqt1/detection_loss‚2.9609533150990806 .loss/FBt1/detection_loss‚3.385986804962158 .loss/FBt4/detection_loss‚5.765496989091237 0loss/FRt4/assignment_loss‚3.7432156801223755åäÌ!“Ñ‚(:ˆ ž“æÔºó¼þ gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.194' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6388 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0t)Y?GÒ‚(:¼ ž“æÔ“î’€) proc.memory.percent‚16.12982749938965 cpu‚25.333589315397692 network.sent‚ 5.753971e+06 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 network.recv‚827167 disk.dm-1.in‚0 disk.sda.out‚ 84.9765625$ disk.nvme0n1.out‚8858.6572265625$ memory_percent‚97.09658622741699& disk.nvme0c0n1.out‚8858.6572265625 disk.sda2.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚298.9471664428711 disk.dm-1.out‚ 84.9765625 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 84.9765625 disk.sda2.out‚0 disk.rbd1.out‚ 11.9912109375 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125' proc.memory.availableMB‚ 475.6953125 disk.drbd2265.in‚0 disk.sda3.in‚0" proc.memory.rssMB‚ 2642.7109375) disk./.usagePercent‚20.06259517336093! disk.dm-0.out‚8858.6572265625 disk.sda.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0?ÀÖv“Ó‚(:ˆ ­“æÔ¿‰ƒý gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.548' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.1096 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0{–\‚GÔ‚(:¼ ­“æÔ•þÞþ disk.rbd1.in‚0 disk.rbd1.out‚ 11.9912109375 cpu‚25.331658688305467 network.recv‚832363 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.sda3.in‚0" disk.dm-0.out‚8924.76708984375 disk.nvme0c0n1.in‚0.078125$ proc.memory.availableMB‚475.6875 disk.sda1.out‚0% disk.nvme0n1.out‚8924.76708984375 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚8924.76708984375 disk.sda3.out‚ 85.7265625 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.drbd2080.out‚0 network.sent‚ 5.861583e+06) disk./.usagePercent‚20.06287217351298 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2080.in‚0" proc.memory.rssMB‚ 2642.7109375 disk.sda.out‚ 85.7265625 disk.dm-1.out‚ 85.7265625 disk.dm-0.in‚0.078125 disk.drbd2265.out‚0 disk.rbd0.in‚0$ memory_percent‚97.09663391113281) proc.memory.percent‚16.12982749938965 proc.cpu.threads‚17$ disk./.usageGB‚298.9512939453125 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0_BÙM2Õ- _wandbruntime‚2275  _runtime‚2275X½˜ÈÖ % _timestamp‚1.7884471598570127e+09 _runtime‚ 2275.32858595 _step‚230 ' lr-AdamW/pg1‚0.00011971641375197476 ' lr-AdamW/pg2‚0.00011971641375197476 trainer/global_step‚2849:çSÍ×Ç ' lr-AdamW/pg2‚0.00011971641375197476 trainer/global_step‚2849 % _timestamp‚1.7884471598570127e+09 _runtime‚ 2275.32858595 _step‚230 ' lr-AdamW/pg1‚0.00011971641375197476æݶ•32Ø- _wandbruntime‚2275  _runtime‚2275TLD2Ù'ÙÓO G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884471598578632e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37762034 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚165263656831488 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53703306 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13658466 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚182998333500416 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2500910503936 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2373865692525 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚167646293150720 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚165263656831488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2381979957248 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚95018058 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53723343 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41314752 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚165263638828544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2382636319232 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚182998510579200 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚160939469179904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚180497600075264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2373211254773 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2381979957248 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚165263638828544 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53703306 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41314754 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37762472 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚163313351911789 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53723343 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13658030 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41314752 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2500757365248 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24104004 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚180497576135168 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚2849 _runtime‚2275.329787879 _step‚231 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41314754 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚167646293150720 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177078784 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚163312680434677 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚160939486219264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚167645618785792 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95038097 IDeviceStatsMonitor.on_train_(°¤Übatch_start/allocation.all.current‚20039 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177078784 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚163312680434677 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚160939486219264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚167645618785792 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚95038097 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚165263656831488 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚53703306 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13658466 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚182998333500416 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2500910503936 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2373865692525 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚167646293150720 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚165263656831488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2381979957248 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚95018058 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53723343 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41314752 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚165263638828544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2382636319232 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚182998510579200 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚160939469179904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚180497600075264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2373211254773 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2381979957248 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚165263638828544 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53703306 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0çÙݬO2Û- _wandbruntime‚2276  _runtime‚2276yºô&â&ÜÜM M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37774684 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚167703465630720 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2374027689737 $ _timestamp‚1.788447160541955e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚165320009870336 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24112174 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚180559250812928 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚180559226872832 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53721760 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚163368401000201 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚160994373310464 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚165319991867392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95050528 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2374682127697 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚232 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41328768 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚165319991867392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95070593 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95070593 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24112172 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2382799385088 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41328770 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2382799385088 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0±Çç&ÝáM M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37774684 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚167703465630720 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2374027689737 $ _timestamp‚1.788447160541955e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚165320009870336 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24112174 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚180559250812928 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚180559226872832 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53721760 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚163368401000201 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚160994373310464 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚165319991867392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95050528 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2374682127697 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚232 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41328768 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚165319991867392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95070593 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95070593 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24112172 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2382799385088 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41328770 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2382799385088 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0èED2Þ- _wandbruntime‚2276  _runtime‚2276ÉB§QßË epoch‚2 trainer/global_step‚2849 0loss/FBt3/assignment_loss‚0.2831398958419011 .loss/FBt3/detection_loss‚1.096980819050523 $ _timestamp‚1.788447160543521e+09 0loss/SRqqt1/assignment_loss‚2.41541118050615 /loss/FRt2/detection_loss‚2.2470333943975733 .loss/FRt3/detection_loss‚1.917705835291633 1loss/SRqqt3/detection_loss‚1.3314527889856926 0loss/SRqqt4/detection_loss‚1.467538467212899 %loss/total_loss‚51.92225566124961 2loss/SRqqt2/assignment_loss‚1.6259786273268135 /loss/FRt2/assignment_loss‚4.149723259692497 0loss/FRt4/assignment_loss‚3.2775160233889307 1loss/SRqqt3/assignment_loss‚1.492438250206984 0loss/FBt2/assignment_loss‚0.3889290130870317 .loss/FRt4/detection_loss‚2.123958989445652 _runtime‚2276.015031762 /loss/FRt3/assignment_loss‚2.520336052885762 2loss/SRqqt4/assignment_loss‚2.0183431908284146 .loss/FRt1/detection_loss‚3.072090883255005 1loss/SRqqt2/detection_loss‚2.0837839408354326 _step‚233 0loss/FBt1/assignment_loss‚0.6494950375209247 1loss/SRqqt1/detection_loss‚2.7583416824539504 .loss/FBt2/detection_loss‚2.758756856123606 /loss/FRt1/assignment_loss‚5.035647401809692 1loss/FBt4/assignment_loss‚0.11428063735365868 .loss/FBt1/detection_loss‚2.534649924437205 .loss/FBt4/detection_loss‚4.558723509311676÷oVàÐ 0loss/FBt3/assignment_loss‚0.2831398958419011 .loss/FBt3/detection_loss‚1.096980819050523 $ _timestamp‚1.788447160543521e+09 0loss/SRqqt1/assignment_loss‚2.41541118050615 /loss/FRt2/detection_loss‚2.2470333943975733 .loss/FRt3/detection_loss‚1.917705835291633 1loss/SRqqt3/detection_loss‚1.3314527889856926 0loss/SRqqt4/detection_loss‚1.467538467212899 %loss/total_loss‚51.92225566124961 2loss/SRqqt2/assignment_loss‚1.6259786273268135 /loss/FRt2/assignment_loss‚4.149723259692497 0loss/FRt4/assignment_loss‚3.2775160233889307 1loss/SRqqt3/assignment_loss‚1.492438250206984 0loss/FBt2/assignment_loss‚0.3889290130870317 .loss/FRt4/detection_loss‚2.123958989445652 _runtime‚2276.015031762 /loss/FRt3/assignment_loss‚2.520336052885762 2loss/SRqqt4/assignment_loss‚2.0183431908284146 .loss/FRt1/detection_loss‚3.072090883255005 1loss/SRqqt2/detection_loss‚2.0837839408354326 _step‚233 0loss/FBt1/assignment_loss‚0.6494950375209247 1loss/SRqqt1/detection_loss‚2.7583416824539504 .loss/FBt2/detection_loss‚2.758756856123606 /loss/FRt1/assignment_loss‚5.035647401809692 1loss/FBt4/assignment_loss‚0.11428063735365868 .loss/FBt1/detection_loss‚2.534649924437205 .loss/FBt4/detection_loss‚4.558723509311676 epoch‚2 trainer/global_step‚2849éFpá‚(:’ ¼“æÔëµþü gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.62' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.724000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0’ðÏKâ‚(:À ¼“æÔ¨¸šÿ cpu‚25.329818393408896 disk.drbd1716.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 12.0234375 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 86.29296875 disk.drbd1716.in‚0 network.recv‚837212 disk.drbd2265.in‚0$ memory_percent‚97.09839820861816' proc.memory.availableMB‚ 475.3984375" proc.memory.rssMB‚ 2642.7109375 network.sent‚ 5.868006e+06' disk.nvme0c0n1.out‚8969.02099609375 disk.rbd0.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 proc.cpu.threads‚17 disk.sda3.in‚0% disk.nvme0n1.out‚8969.02099609375* disk./.usagePercent‚20.063657092612956$ disk./.usageGB‚298.9629898071289 disk.sda3.out‚ 86.29296875 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.dm-1.in‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.sda.out‚ 86.29296875 disk.nvme0c0n1.in‚0.078125" disk.dm-0.out‚8969.02099609375 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0) proc.memory.percent‚16.12982749938965áÝg·“ã‚(:ˆ Ë“æÔ…Àôü gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.229' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4458 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Srg Qä‚(:½ Ë“æÔý§èÿ disk.drbd2080.in‚0' disk.nvme0c0n1.out‚9017.56396484375 network.recv‚842148" disk.dm-0.out‚9017.56396484375 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 86.96484375 proc.cpu.threads‚17 disk.dm-1.out‚ 86.96484375 disk.sda3.in‚0 disk.drbd2080.out‚0 disk.«Í÷drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 12.1484375 disk.drbd1716.out‚0 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125' proc.memory.availableMB‚ 475.5390625 cpu‚25.327998793133307 disk.drbd2266.in‚0 disk.sda.out‚ 86.96484375 disk.dm-1.in‚0 disk.drbd1716.in‚0# memory_percent‚97.0975399017334 network.sent‚ 5.975618e+06( disk./.usagePercent‚20.0639609735561 disk.drbd2265.out‚0" proc.memory.rssMB‚ 2642.7109375) proc.memory.percent‚16.12982749938965$ disk./.usageGB‚298.9675178527832 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.in‚0% disk.nvme0n1.out‚9017.56396484375yþ ‘å‚(:† Ú“æÔñÿ•ý gpu.0.gpu‚35.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚131.77' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.354 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Óa Dæ‚(:¹ Ú“æÔ߀Áÿ disk.drbd1716.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.dm-1.out‚87.78125" proc.memory.rssMB‚ 2642.7109375 network.sent‚ 5.982261e+06 disk.rbd0.in‚0 disk.sda1.out‚0 proc.cpu.threads‚17 disk.sda1.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 12.2109375' disk.nvme0c0n1.out‚9061.75146484375 network.recv‚847214* disk./.usagePercent‚20.064894633033475 disk.drbd2266.out‚0$ memory_percent‚97.09799289703369 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚87.78125 disk.drbd2266.in‚0 disk.sda.out‚87.78125 disk.sda2.in‚0 disk.dm-1.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125) proc.memory.percent‚16.12982749938965 disk.sda3.in‚0 disk.sda2.out‚0% disk.nvme0n1.out‚9061.75146484375% disk./.usageGB‚298.98143005371094 disk.drbd1716.in‚0 disk.drbd2265.out‚0" disk.dm-0.out‚9061.75146484375( proc.memory.availableMB‚ 475.46484375 disk.rbd0.out‚0 disk.nvme0n1.in‚0.078125 cpu‚25.326222951389795yç32ç- _wandbruntime‚2312  _runtime‚2312¢««mÉèà trainer/global_step‚2899 % _timestamp‚1.7884471967062814e+09 _runtime‚2312.177980151 _step‚234 ' lr-AdamW/pg1‚0.00011929356779901059 ' lr-AdamW/pg2‚0.00011929356779901059š9eÎéÈ _runtime‚2312.177980151 _step‚234 ' lr-AdamW/pg1‚0.00011929356779901059 ' lr-AdamW/pg2‚0.00011929356779901059 trainer/global_step‚2899 % _timestamp‚1.7884471967062814e+09êûHM2ê- _wandbruntime‚2312  _runtime‚2312ù?©Ù'ëÓO E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚170504930898944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38398989 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2422965763072 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2543990997504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42015552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54629358 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚170504256534016 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚2899 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24512406 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13886583 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚168081290770944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 _step‚235 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚168081308773888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 198050304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2422965763072 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54649395 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚163684692747264 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42015554 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54649395 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2414701031933 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 198050304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2422965763072 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54649395 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚163684692747264 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42015554 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54649395 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2414701031933 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42015552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54629358 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚170504256534016 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚2899 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24512406 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13886583 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚168081290770944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 _step‚235 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚168081308773888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚96697445 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚170562103378944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚236 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185454080 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚183641787696128 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513984 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚166154442867609 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2544687357440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2414863029145 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38411332 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚2899 _runtime‚2312.908736485 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96697445 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2415517467105 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2423785190912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13891205 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚186186636567552 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚168137643809792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚186186451113472 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2544848871424 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54647812 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24520574 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚170561429000704 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚168137661812736 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42029568 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚166155114344929 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42029568 QCDeviceStatsMonitor.on_tr— pvàain_batch_end/inactive_split_bytes.all.peak‚ 336208384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚163739596877824h³zøè&ïâM JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚163739596877824 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚54667875 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚170562103378944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚168137643809792 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚96697445 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚170562103378944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚236 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185454080 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚183641787696128 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513984 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚166154442867609 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2544687357440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2414863029145 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38411332 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚2899 _runtime‚2312.908736485 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96697445 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2415517467105 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2423785190912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13891205 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚186186636567552 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚168137643809792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚186186451113472 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2544848871424 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54647812 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24520574 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚170561429000704 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚168137661812736 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42029568 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚166155114344929 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42029568 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928ìÿúѱ2ð- _wandbruntime‚2312  _runtime‚2312ƒ©ž°WñÑ 0loss/FBt4/assignment_loss‚0.2315518584684469 1loss/SRqqt2/detection_loss‚2.1337161330800307 .loss/FBt4/detection_loss‚4.909757888317108 2loss/SRqqt2/assignment_loss‚1.7226061075925827 0loss/FBt2/assignment_loss‚0.5619178949855155 1loss/SRqqt1/detection_loss‚2.8916749386116862 1loss/SRqqt4/detection_loss‚1.8276640855922148 /loss/FBt1/detection_loss‚3.5383167597982617 _step‚237 /loss/FRt1/assignment_loss‚4.349581175837024 /loss/FRt4/assignment_loss‚4.057581121271307 /loss/FRt3/detection_loss‚1.7316094666719437 /loss/FRt2/assignment_loss‚4.500235018134117 /loss/FRt3/assignment_loss‚2.514182483066212 .loss/FRt1/detection_loss‚2.917672272386222 /loss/FRt4/detection_loss‚2.4083274101669137 epoch‚2 trainer/global_step‚2899 _runtime‚2312.909942231 /loss/FRt2/detection_loss‚2.2771451592445375 % _timestamp‚1.7884471974383645e+09 %loss/total_loss‚53.18841068110133 0loss/SRqqt1/assignment_loss‚2.26050441339612 1loss/FBt1/assignment_loss‚0.22762254309327545 1loss/SRqqt3/detection_loss‚1.3283348632976413 /loss/FBt2/detection_loss‚2.9925972506403924 .loss/FBt3/detection_loss‚0.746265772630771 2loss/SRqqt3/assignment_loss‚1.3043025025399402 1loss/SRqqt4/assignment_loss‚1.571499766065524 1loss/FBt3/assignment_loss‚0.18374379621354536%LJZ\òÖ %loss/total_loss‚53.18841068110133 0loss/SRqqt1/assignment_loss‚2.26050441339612 1loss/FBt1/assignment_loss‚0.22762254309327545 1loss/SRqqt3/detection_loss‚1.3283348632976413 /loss/FBt2/detection_loss‚2.9925972506403924 .loss/FBt3/detection_loss‚0.746265772630771 2loss/SRqqt3/assignment_loss‚1.3043025025399402 1loss/SRqqt4/assignment_loss‚1.571499766065524 1loss/FBt3/assignment_loss‚0.18374379621354536 0loss/FBt4/assignment_loss‚0.2315518584684469 1loss/SRqqt2/detection_loss‚2.1337161330800307 .loss/FBt4/detection_loss‚4.909757888317108 2loss/SRqqt2/assignment_loss‚1.7226061075925827 0loss/FBt2/assignment_loss‚0.5619178949855155 1loss/SRqqt1/detection_loss‚2.8916749386116862 1loss/SRqqt4/detection_loss‚1.8276640855922148 /loss/FBt1/detection_loss‚3.5383167597982617 _step‚237 /loss/FRt1/assignment_loss‚4.349581175837024 /loss/FRt4/assignment_loss‚4.057581121271307 /loss/FRt3/detection_loss‚1.7316094666719437 /loss/FRt2/assignment_loss‚4.500235018134117 /loss/FRt3/assignment_loss‚2.514182483066212 .loss/FRt1/detection_loss‚2.917672272386222 /loss/FRt4/detection_loss‚2.4083274101669137 epoch‚2 trainer/global_step‚2899 _runtime‚2312.909942231 /loss/FRt2/detection_loss‚2.2771451592445375 % _timestamp‚1.7884471974383645e+09íî½Àx“ó‚(:ˆ é“æÔÓŠ…ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.341' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6682 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÉGG>=ô‚(:² é“æÔ³«¥ÿ' disk.nvme0c0n1.out‚9101.53466796875 disk.rbd0.in‚0 cpu‚25.32446780706046 network.sent‚ 5.987425e+06 disk.sda3.out‚88.21875" disk.dm-0.out‚9101.53466796875 disk.sda.in‚0% disk.nvme0n1.out‚9101.53466796875 disk.dm-1.in‚0$ memory_percent‚97.09739685058594) disk./.usagePercent‚20.06522027461703 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 network.recv‚850480 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda2.in‚0 disk.dm-1.out‚88.21875 disk.drbd1716.in‚0 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2265.out‚0) proc.memory.percent‚16.12982749938965$ disk./.usageGB‚298.9862823486328 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0$ proc.memory.availableMB‚475.5625 disk.sda2.out‚0 disk.rbd1.out‚ 12.2109375 disk.nvme0n1.in‚0.078125 disk.sda.out‚88.21875 disk.sda1.in‚0" proc.memory.rssMB‚ 2642.7109375hŸãù’õ‚(:‡ ø“æÔîçöü gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚125.693' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.1386 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0™­ÈÌLö‚(:Á ø“æԻȕÿ disk.sda2.out‚0 disk.dm-1.out‚ 88.84765625 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda.out‚ 88.84765625 disk.sda1.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 88.84765625 disk.rbd0.in‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚9145.09326171875 disk.drbd1716.out‚0 disk.sda3.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0' proc.memory.availableMB‚ 475.6640625 disk.rbd1.out‚ 12.2431640625 disk.dm-1.in‚0" disk.dm-0.out‚9145.09326171875" proc.memory.rssMB‚ 2642.7109375 network.recv‚857477 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚9145.09326171875) disk./.usagePercent‚20.06590253469578$ disk./.usageGB‚298.9964485168457 disk.rbd0.out‚0) proc.memory.percent‚16.12982749938965 cpu‚25.32273286372903 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.sda1.in‚0$ memory_percent‚97.09677696228027 network.sent‚ 6.096372e+06” Oþ2÷- _wandbruntime‚2349  _runtime‚2349 wùÉøà _runtime‚2349.480032563 _step‚238 ' lr-AdamW/pg1‚0.00011885906179956313 ' lr-AdamW/pg2‚0.00011885906179956313 trainer/global_step‚2949 % _timestamp‚1.7884472340083728e+09§`ö¼ÎùÈ ' lr-AdamW/pg1‚0.00011885906179956313 ' lr-AdamW/pg2‚0.00011885906179956313 trainer/global_step‚2949 % _timestamp‚1.7884472340083728e+09 _runtime‚2349.480032563 _step‚238îþ€2ú- _wandbruntime‚2349  _runtime‚23494ÉÙ'ûÓO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42716354 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42716352 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172013056 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55575447 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39036066 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24920804 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98271762 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42716354 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2586800247808 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚173363568647168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39035596 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42716352 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚170898942713344 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚189249646102528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚189249450149376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚168885435646605 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55555410 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2463951568896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚468 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚173362894282240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14115260 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2463951568896 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚173362894282240 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2586972260864 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚2949 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2464607930880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚186662673841664 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55555410 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚173363568647168 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚166429899275264 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚166429882235904 _runtime‚2349.480847031 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚170898960716288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14114792 % _timestamp‚1.7884472340090559e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _step‚239 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2464607930880 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414zö8/hüØO IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55575447 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39036066 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24920804 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚98271762 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42716354 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2586800247808 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚173363568647168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39035596 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42716352 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚170898942713344 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚189249646102528 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚189249450149376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚168885435646605 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55555410 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2463951568896 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 QIDeviceStatv¤vsMonitor.on_train_batch_start/inactive_split.small_pool.current‚468 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚173362894282240 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14115260 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2463951568896 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚173362894282240 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2586972260864 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚2949 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2464607930880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚186662673841664 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55555410 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚173363568647168 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚166429899275264 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚166429882235904 _runtime‚2349.480847031 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚170898960716288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14114792 % _timestamp‚1.7884472340090559e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _step‚239 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2464607930880 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42716354 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42716352 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172013056 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872ïbØÈ2ý- _wandbruntime‚2350  _runtime‚2350· ž‹â&þÜM P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2464770996736 trainer/global_step‚2949 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚170955295752192 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚170955295752192 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14119376 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚98304232 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2465427372032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚173420066748928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98304232 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55573864 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 171999744 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14119846 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2455698368553 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42730368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚186724324579328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _step‚240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚173420741127168 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98324297 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2587822788608 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2464770996736 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2456352806513 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚186724300639232 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚166484803405824 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55593927 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39048348 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24928972 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55593927 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚170955313755136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2587650788864 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55573864 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39048820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚470 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24928974 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42730370‹Ô1ç&ÿáM 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚170955295752192 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚170955295752192 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14119376 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚98304232 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2465427372032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚173420066748928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98304232 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55573864 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 171999744 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14119846 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2455698368553 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42730368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚186724324579328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _step‚240 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚173420741127168 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98324297 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2587822788608 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2464770996736 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2456352806513 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚186724300639232 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚166484803405824 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55593927 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39048348 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24928972 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55593927 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚170955313755136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2587650788864 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55573864 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39048820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚470 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24928974 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42730370 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2464770996736 trainer/global_step‚2949ðu‹Ø2€- _wandbruntime‚2350  _runtime‚2350‘ë’SÍ .loss/FRt2/assignment_loss‚4.48584465864228 0loss/FRt4/assignment_loss‚3.5291519793597135 2loss/SRqqt3/assignment_loss‚1.4056413566221053 /loss/FRt3/detection_loss‚1.8173701340501958 /loss/FRt4/detection_loss‚2.2794784009456635 /loss/FBt2/detection_loss‚1.8022884756326676 /loss/FBt4/detection_loss‚3.1747997403144836 0loss/FRt3/assignment_loss‚2.5114586049860175 1loss/SRqqt4/detection_loss‚1.6560013192264658 /loss/FBt3/detection_loss‚0.8994713897424701 $ _timestamp‚1.788447234728067e+09 -loss/FRt2/detection_loss‚2.58146284266216 1loss/SRqqt1/assignment_loss‚2.593461406953407 2loss/SRqqt4/assignment_loss‚1.9253472915866918 0loss/SRqqt1/detection_loss‚2.580315239501722 1loss/SRqqt2/detection_loss‚1.9194472472776065 epoch‚2 /loss/FRt1/detection_loss‚3.2185469937324522 _step‚241 0loss/FBt3/assignment_loss‚0.2603801831425924 %loss/total_loss‚51.23080713404394 trainer/global_step‚2949 .loss/FBt1/detection_loss‚3.481277100245158 /loss/FBt4/assignment_loss‚0.394771753148234 .loss/FRt1/assignment_loss‚4.61018364906311 2loss/SRqqt2/assignment_loss‚1.8592121851715175 0loss/FBt2/assignment_loss‚0.6484385419636964 0loss/FBt1/assignment_loss‚0.3674290323034256 1loss/SRqqt3/detection_loss‚1.2290276077701086 _runtime‚ 2350.19955212|°"X‚Ò .loss/FBt1/detection_loss‚3.481277100245158 /loss/FBt4/assignment_loss‚0.394771753148234 .loss/FRt1/assignment_loss‚4.61018364906311 2loss/SRqqt2/assignment_loss‚1.8592121851715175 0loss/FBt2/assignment_loss‚0.6484385419636964 0loss/FBt1/assignment_loss‚0.3674290323034256 1loss/SRqqt3/detection_loss‚1.2290276077701086 _runtime‚ 2350.19955212 .loss/FRt2/assignment_loss‚4.48584465864228 0loss/FRt4/assignment_loss‚3.5291519793597135 2loss/SRqqt3/assignment_loss‚1.4056413566221053 /loss/FRt3/detection_loss‚1.8173701340501958 /loss/FRt4/detection_loss‚2.2794784009456635 /loss/FBt2/detection_loss‚1.8022884756326676 /loss/FBt4/detection_loss‚3.1747997403144836 0loss/FRt3/assignment_loss‚2.5114586049860175 1loss/SRqqt4/detection_loss‚1.6560013192264658 /loss/FBt3/detection_loss‚0.8994713897424701 $ _timestamp‚1.788447234728067e+09 -loss/FRt2/detection_loss‚2.58146284266216 1loss/SRqqt1/assignment_loss‚2.593461406953407 2loss/SRqqt4/assignment_loss‚1.9253472915866918 0loss/SRqqt1/detection_loss‚2.580315239501722 1loss/SRqqt2/detection_loss‚1.9194472472776065 epoch‚2 /loss/FRt1/detection_loss‚3.2185469937324522 _step‚241 0loss/FBt3/assignment_loss‚0.2603801831425924 %loss/total_loss‚51.23080713404394 trainer/global_step‚2949ñåhÍ“ƒ‚(:ˆ ‡”æÔÏâ’ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.978' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5956 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0áÁçÐL„‚(:Á ‡”æÔ€©€ disk.drbd1716.out‚0 disk.sda3.out‚ 89.2265625* disk./.usagePercent‚20.066600155226585% disk.nvme0n1.out‚9203.83056640625 disk.dm-1.in‚0 disk.dm-1.out‚ 89.2265625 cpu‚25.32089114973792 network.recv‚860665 disk.sda2.in‚0 disk.dm-0.in‚0.078125) proc.memory.percent‚16.12982749938965 proc.cpu.threads‚17 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 475.16796875 disk.sda1.out‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 network.sent‚ 6.101506e+06 disk.rbd1.out‚ 12.3681640625 disk.drbd2265.in‚0 disk.drbd2080.in‚0$ memory_percent‚97.09980487823486 disk.sda.in‚0' disk.nvme0c0n1.out‚9203.83056640625 disk.drbd2266.in‚0" disk.dm-0.out‚9203.83837890625" proc.memory.rssMB‚ 2642.7109375% disk./.usageGB‚299.00684356689453 disk.sda.out‚ 89.2265625=½[“…‚(:ˆ –”æÔÇÍ”ý gpu.0.gpu‚39.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.447' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.6894 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0=ÕJ€O†‚(:Ä –”æÔãÜðþ% disk./.usageGB‚298.99008560180664 disk.rbd1.out‚ 12.4306640625 disk.sda3.out‚ 89.83984375 disk.drbd2266.out‚0$ memory_percent‚97.09963798522949* disk./.usagePercent‚20.065475514128668 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚9248.03369140625 disk.drbd2080.in‚0 disk.sda.out‚ 89.83984375" proc.memory.rssMB‚ 2642.7109375 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.sda1.in‚0' proc.memory.availableMB‚ 475.1953125 cpu‚25.319224749789864 network.sent‚ 6.210819e+06 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚9248.03369140625 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 network.recv‚867554 disk.dm-1.in‚0 disk.dm-1.out‚ 89.83984375" disk.dm-0.out‚9248.03369140625 disk.sda.in‚0 disk.sda2.in‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 proc.cpu.threads‚17 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.sda1.out‚0) proc.memory.percent‚16.12982749938965pAöž‡‚(:“ ¥”æÔµýý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.593' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.718599999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Å„óm?ˆ‚(:´ ¥”æÔ™±£ÿ disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 network.recv‚871297 disk.rbd0.in‚0$ memory_percent‚97.10197448730469 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.dm-1.out‚90.46875 disk.sda3.out‚90.46875" proc.memory.rssMB‚ 2642.7109375 proc.cpu.threads‚17% disk./.usageGB‚298.99969482421875! disk.dm-0.out‚9298.5341796875 disk.sda1.out‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.rbd1.out‚ 12.4306640625 disk.nvme0n1.in‚0.078125& disk.nvme0c0n1.out‚9298.5341796875 network.sent‚ 6.216131e+06) disk./.usagePercent‚20.06612039710742 disk.sda.out‚90.46875$ proc.memory.availableMB‚474.8125 disk.rbd0.out‚0$ disk.nvme0n1.out‚9298.5341796875 disk.sda3.in‚0) proc.memory.percent‚16.12982749938965 cpu‚25.317439736284584 disk.drbd1716.out‚0 disk.sda.in‚0 disk.dm-1.in‚0†Š †2‰- _wandbruntime‚2386  _runtime‚2386æMyuÉŠÃ _runtime‚2386.139136257 _step‚242 ' lr-AdamW/pg1‚0.00011841298630403141 ' lr-AdamW/pg2‚0.00011841298630403141 trainer/global_step‚2999 % _timestamp‚1.7884472706675508e+09Þ$UÎ‹È _step‚242 ' lr-AdamW/pg1‚0.00011841298630403141 ' lr-AdamW/pg2‚0.00011841298630403141 trainer/global_step‚2999 % _timestamp‚1.7884472706675508e+09 _runtime‚2386.139136257ò Y2Œ- _wandbruntime‚2386  _runtime‚2386ǪÐO›ÑO M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚99918653 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43417152 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚169175105803264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pï+NÌ<&ool.allocated‚189745210724864 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚173716612658688 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2505593736704 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚176222206395392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43417154 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163624448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 2386.14017636 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2504937374720 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39672715 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564544 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚2999 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2504937374720 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2629701062656 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56501499 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚173716612658688 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚171671477514013 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56481462 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚169175088763904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚243 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39672286 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25329204 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2495717272997 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚173716594655744 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14343509 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14343082 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2505593736704 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚176221532030464 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚171670806036901 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚176222206395392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚176221532030464 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2496371710749 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43417152 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56481462 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43417154 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 2386.14017636 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2504937374720 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39672715 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564544 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚2999 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2504937374720 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2629701062656 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56501499 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚173716612658688 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚171671477514013 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56481462 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚169175088763904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚243 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39672286 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25329204 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2495717272997 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚173716594655744 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14343509 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14343082 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2505593736704 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚176221532030464 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚171670806036901 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚176222206395392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚176221532030464 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2496371710749 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43417152 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56481462 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088ónd92- _wandbruntime‚2386  _runtime‚2386ë9¾“â&ÜM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2506413177856 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165708288 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2497188145921 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚ 2386.81486305 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2506413177856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚189806837522432 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39685119 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚192437400180224 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2630728366080 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚99931084 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56519979 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99951149 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚176279378875392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43431170 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚171726526602425 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚173772947694592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43431168 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚176279378875392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚192437589828608 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99951149 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56499916 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43431168 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2505756802560 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39685554 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2496533707961 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14347747 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43431170 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚080 ÿ± ‘áM S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚171726526602425 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚173772947694592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43431168 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚176279378875392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚192437589828608 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99951149 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56499916 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43431168 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2505756802560 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39685554 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2496533707961 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14347747 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43431170 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2506413177856 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165708288 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2497188145921 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚ 2386.81486305 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2506413177856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚189806837522432 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39685119 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚192437400180224 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2630728366080 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚99931084 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56519979 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99951149 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚176279378875392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43431170ô‘LÊõ2’- _wandbruntime‚2386  _runtime‚2386Ä(΃W“Ñ 0loss/FBt1/assignment_loss‚0.4509215519642112 0loss/FBt2/assignment_loss‚0.5956288415444675 /loss/FRt3/detection_loss‚1.9489962838590145 .loss/FRt4/detection_loss‚2.284421812813237 1loss/SRqqt2/detection_loss‚2.1281791044907137 0loss/FRt1/assignment_loss‚4.9028914857793735 2loss/SRqqt4/assignment_loss‚1.7510033344680613 0loss/FBt3/assignment_loss‚0.2506806025143488 1loss/SRqqt4/detection_loss‚1.8183560143817554 .loss/FBt4/detection_loss‚4.104626879096031 trainer/global_step‚2999 _step‚245 0loss/FRt4/assignment_loss‚3.6153622145922677 /loss/FRt1/detection_loss‚3.0025959368105286 /loss/FBt3/detection_loss‚0.9642833920026367 %loss/total_loss‚52.90668803429824 _runtime‚2386.816038129 /loss/FBt2/detection_loss‚2.4476533690467477 .loss/FRt2/detection_loss‚2.202193317460079 /loss/FRt2/assignment_loss‚4.183486725769791 1loss/SRqqt1/assignment_loss‚2.499366033461786 1loss/SRqqt3/assignment_loss‚1.421118560181537 1loss/FBt4/assignment_loss‚0.49110573797952384 1loss/SRqqt3/detection_loss‚1.2524418942105602 $ _timestamp‚1.788447271344515e+09 1loss/SRqqt1/detection_loss‚2.7777907617630495 .loss/FBt1/detection_loss‚3.624344605207443 epoch‚2 0loss/FRt3/assignment_loss‚2.2659402281045913 1loss/SRqqt2/assignment_loss‚1.923299346796491µ^å\”Ö $ _timestamp‚1.788447271344515e+09 1loss/SRqqt1/detection_loss‚2.7777907617630495 .loss/FBt1/detection_loss‚3.624344605207443 epoch‚2 0loss/FRt3/assignment_loss‚2.2659402281045913 1loss/SRqqt2/assignment_loss‚1.923299346796491 0loss/FBt1/assignment_loss‚0.4509215519642112 0loss/FBt2/assignment_loss‚0.5956288415444675 /loss/FRt3/detection_loss‚1.9489962838590145 .loss/FRt4/detection_loss‚2.284421812813237 1loss/SRqqt2/detection_loss‚2.1281791044907137 0loss/FRt1/assignment_loss‚4.9028914857793735 2loss/SRqqt4/assignment_loss‚1.7510033344680613 0loss/FBt3/assignment_loss‚0.2506806025143488 1loss/SRqqt4/detection_loss‚1.8183560143817554 .loss/FBt4/detection_loss‚4.104626879096031 trainer/global_step‚2999 _step‚245 0loss/FRt4/assignment_loss‚3.6153622145922677 /loss/FRt1/detection_loss‚3.0025959368105286 /loss/FBt3/detection_loss‚0.9642833920026367 %loss/total_loss‚52.90668803429824 _runtime‚2386.816038129 /loss/FBt2/detection_loss‚2.4476533690467477 .loss/FRt2/detection_loss‚2.202193317460079 /loss/FRt2/assignment_loss‚4.183486725769791 1loss/SRqqt1/assignment_loss‚2.499366033461786 1loss/SRqqt3/assignment_loss‚1.421118560181537 1loss/FBt4/assignment_loss‚0.49110573797952384 1loss/SRqqt3/detection_loss‚1.2524418942105602õªN‹N’•‚(:‡ ´”æÔ꜅ý gpu.0.gpu‚36.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.655' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.731 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0\¨;E–‚(:º ´”æÔÏâ›ÿ disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.in‚0 disk.dm-1.out‚ 91.328125 disk.drbd2266.in‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125" proc.memory.rssMB‚ 2642.7109375 network.sent‚ 6.222526e+06 network.recv‚876073 disk.sda3.in‚0 disk.sda3.out‚ 91.328125) proc.memory.percent‚16.12982749938965 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.drbd2080.in‚0& disk.nvme0c0n1.out‚9348.6611328125 disk.rbd0.out‚0 disk.sda1.out‚0 disk.rbd1.out‚ 12.462890625( proc.memory.availableMB‚ 474.74609375 cpu‚25.31578962772456* disk./.usagePercent‚20.066148045921118 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0$ memory_percent‚97.10237979888916 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.drbd2265.in‚0 disk.rbd1.in‚0$ disk.nvme0n1.out‚9348.6611328125% disk./.usageGB‚299.00010681152344 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125! disk.dm-0.out‚9348.6611328125 disk.sda.out‚ 91.328125¶ÈB¾“—‚(:ˆ ÔæÔô ƒý gpu.0.gpu‚35.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.789' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.1578 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0£ªyrL˜‚(:Á ÔæÔâÓ¢ÿ network.recv‚881225 disk.rbd1.out‚ 12.673828125 disk.dm-1.out‚ 92.02734375 disk.drbd2080.in‚0 disk.sda.in‚0 network.sent‚ 6.330199e+06' disk.nvme0c0n1.out‚9396.44677734375 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2266.out‚0 disk.sda.out‚ 92.02734375) proc.memory.percent‚16.12982749938965 disk.drbd2080.out‚0$ memory_percent‚97.10237979888916( proc.memory.availableMB‚ 474.74609375 proc.cpu.threads‚17$ disk./.usageGB‚299.0102767944336 disk.sda2.in‚0% disk.nvme0n1.out‚9396.44677734375 disk.dm-1.in‚0 disk.sda3.in‚0 disk.rbd0.out‚0" disk.dm-0.out‚9396.44677734375( disk./.usagePercent‚20.0668305620074 disk.sda2.out‚0 disk.rbd1.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 cpu‚25.314157366406402 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 92.02734375 disk.rbd0.in‚0" proc.memory.rssMB‚ 2642.7109375ÿ±{ 2™- _wandbruntime‚2422  _runtime‚2422Æ6´¾ÉšÃ ' lr-AdamW/pg1‚0.00011795543427388011 ' lr-AdamW/pg2‚0.00011795543427388011 trainer/global_step‚3049 % _timestamp‚1.7884473073932433e+09 _runtime‚2422.864825772 _step‚246«×VÎ›È _step‚246 ' lr-AdamW/pg1‚0.00011795543427388011 ' lr-AdamW/pg2‚0.00011795543427388011 trainer/global_step‚3049 % _timestamp‚1.7884473073932433e+09 _runtime‚2422.864825772öc(ÐÓ2œ- _wandbruntime‚2422  _runtime‚2422•uÚrÝ'×O ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚195500684372480 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚171920312331264 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 101545505 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163624448 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚44117952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25737606 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2537207050157 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚176534264601088 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44117952 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57407514 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7884473073939755e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚179080169778688 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57427551 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 101525466 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25737604 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14571500 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚195500496807936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚179080169778688 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57427551 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚179080844143616 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2545923180544 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14571950 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚176534246598144 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564544 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚176534264601088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2672773139968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2672936764416 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2545923180544 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44117954 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚171920295291904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2536552612405 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚179080169778688 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57427551 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚179080844143616 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2545923180544 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚14571950 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚176534246598144 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564544 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚176534264601088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2672773139968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2672936764416 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2545923180544 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚44117954 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚171920295291904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2536552612405 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚176534264601088 WJDeviceStatsMonËÁ;íitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44117952 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚57407514 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7884473073939755e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚179080169778688 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57427551÷ÇT)‹2Ÿ- _wandbruntime‚2423  _runtime‚2423Ï l{æ& àM QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 $ _timestamp‚1.788447308093833e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101578001 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40322241 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14576018 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57425968 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚192889398345728 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2673791803904 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚174513239947153 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44131970 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2547398983680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2673621901312 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚176590599636992 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 101557936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25745774 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚195562996306944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2538023485329 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44131968 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57446031 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14576467 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚179138016623616 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚176590617639936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2546742608384 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40321790 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚176590599636992 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚451 _runtime‚2423.565977743 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2546742608384 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101578001 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚179137342245376 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44131970 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚171975199422464 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44131968 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2537369047369 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _step‚248 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚449 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚174512568469833 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57446031 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2547398983680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844ÏŽ˜›ë&¡åM M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚449 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚174512568469833 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57446031 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2547398983680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 $ _timestamp‚1.788447308093833e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101578001 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40322241 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚14576018 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57425968 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚192889398345728 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2673791803904 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚174513239947153 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44131970 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2547398983680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2673621901312 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚176590599636992 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 101557936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25745774 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚195562996306944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902592 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2538023485329 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44131968 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57446031 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14576467 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚179138016623616 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚176590617639936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2546742608384 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40321790 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚176590599636992 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚451 _runtime‚2423.565977743 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2546742608384 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101578001 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚179137342245376 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44131970 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚171975199422464 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44131968 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2537369047369 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _step‚248 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2587565348352 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚3099 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2586908986368 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14798700 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2577387951813 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58333566 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚179351916543488 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚181938807526912 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2715969528320 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44818752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2587565348352 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚3099 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2586908986368 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚14798700 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2577387951813 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58333566 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚179351916543488 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚181938807526912 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2715969528320 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44818752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0ûcœž¯‚(:“ ð”æÔó•šý gpu.0.gpu‚44.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚127.676' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.535200000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0]ß9D°‚(:¹ ð”æÔÝ¡†€ disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 13.064453125$ disk.nvme0n1.out‚9575.2841796875# memory_percent‚95.5437421798706 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 93.890625 disk.rbd0.in‚0 disk.drbd1716.out‚0 disk.rbd0.out‚0 disk.sda1.in‚0 network.sent‚ 6.450967e+06 network.recv‚896327 disk.drbd2266.in‚0 disk.sda.out‚ 93.890625 disk.dm-1.in‚0% disk./.usageGB‚299.04662322998047 disk.sda2.in‚0 disk.dm-1.out‚ 93.890625 disk.drbd1716.in‚0) proc.memory.percent‚16.12982749938965 cpu‚25.30926522212362* disk./.usagePercent‚20.069269801793645 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 proc.cpu.threads‚17 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125( proc.memory.availableMB‚ 730.11328125" proc.memory.rssMB‚ 2642.7109375& disk.nvme0c0n1.out‚9575.2841796875! disk.dm-0.out‚9575.2841796875 disk.sda1.out‚0E´Ç2±- _wandbruntime‚2460  _runtime‚2460u›Û—ç&²áM H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2588384789504 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2578858824737 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14803732 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚177298610337241 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103204853 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2716811097088 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚2460.345142391 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58372083 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40957906 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚179408251579392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58372083 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58352020 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036992 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚181995979993600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40957449 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103204853 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44832768 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58352020 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚179408251579392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44832770 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚3099 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26154174 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚174720405950464 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚195971911288832 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2716985193984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26154172 % _timestamp‚1.7884473448728962e+09#€Ü³æM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚181996654371840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096896 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚174720405950464 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚195971911288832 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2716985193984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26154172 % _timestamp‚1.7884473448728962e+09 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2588384789504 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2578858824737 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚14803732 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚177298610337241 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytesw¯™Ï.large_pool.peak‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103204853 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2716811097088 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚2460.345142391 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58372083 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40957906 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚179408251579392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58372083 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58352020 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036992 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚181995979993600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40957449 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103204853 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44832768 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58352020 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚179408251579392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44832770 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚3099 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26154174 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2628551154176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚184797445275136 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41581240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚201752076762624 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15026836 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 $ _timestamp‚1.788447381363662e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2759093860864 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚180029603116237 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2628551154176 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41581680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚177410725387264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚3149 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚201751891295232 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2618223291221 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45519554 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15027274 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104799209 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚198992797434368 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _runtime‚2496.835535211 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚182169568485888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚182169568485888 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59259618 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚198992821374464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚184797445275136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚184798119640064 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2627894792192 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚438 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59279655 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45519554 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2627894792192 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59259618 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185467392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚184798119640064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2Îcoá'ÀÛO UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚41581240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚201752076762624 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15026836 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 $ _timestamp‚1.788447381363662e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2759093860864 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚180029603116237 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2628551154176 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41581680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚177410725387264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚3149 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚201751891295232 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2618223291221 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45519554 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15027274 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104799209 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚198992797434368 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _runtime‚2496.835535211 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚182169568485888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚182169568485888 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59259618 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚198992821374464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚184797445275136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚184798119640064 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2627894792192 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚438 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59279655 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45519554 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2627894792192 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59259618 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185467392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚184798119640064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2628551154176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚184797445275136 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2629370595328 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45533568 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104831705 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104811640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59298135 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.OtJ®current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚182225921524736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41593875 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚201814589704704 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104831705 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2759953981440 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚177465629517824 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚184854617741824 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2628714220032 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104811640 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚201814402153472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚180084652204649 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚450 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚2497.524652725 _step‚256 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚184855292120064 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚182225903521792 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45533568 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59278072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚180085323681969 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41594327 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚452 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15031753 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15031303 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚199054448172032 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2619694164145 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚184854617741824 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2628714220032 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104811640 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚201814402153472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚180084652204649 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚450 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚2497.524652725 _step‚256 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚184855292120064 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚182225903521792 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45533568 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59278072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚180085323681969 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41594327 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚452 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15031753 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15031303 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚199054448172032 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2619694164145 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2629370595328 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45533568 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104831705 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104811640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59298135 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚182225921524736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41593875 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚201814589704704 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104831705 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2759953981440 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0€ù”2Ä- _wandbruntime‚2497  _runtime‚2497ÌF]¼ZÅÔ 0loss/FRt4/assignment_loss‚3.9562379227744207 2loss/SRqqt1/assignment_loss‚2.5941291550795236 1loss/SRqqt2/detection_loss‚1.6939995139837265 /loss/FBt1/detection_loss‚3.4858792573213577 .loss/FBt2/detection_loss‚2.307929197947184 1loss/SRqqt4/assignment_loss‚1.558844307773426 /loss/FBt3/detection_loss‚0.6716481881081372 0loss/FRt2/assignment_loss‚4.9159549077351885 2loss/SRqqt2/assignment_loss‚1.8340198722978434 1loss/SRqqt4/detection_loss‚1.8032228521636275 .loss/FBt4/detection_loss‚3.387626016139984 _step‚257 /loss/FRt1/assignment_loss‚5.498709829648336 2loss/SRqqt3/assignment_loss‚1.2969095366733039 /loss/FRt3/detection_loss‚1.9365627033015091 0loss/FBt2/assignment_loss‚0.4529741532933258 1loss/FBt3/assignment_loss‚0.27332194256492065 /loss/FRt2/detection_loss‚2.0455879156003918 1loss/SRqqt1/detection_loss‚2.7019836008548737 %loss/total_loss‚52.55957688395345 epoch‚2 trainer/global_step‚3149 /loss/FRt3/assignment_loss‚2.476622256139914 0loss/FBt1/assignment_loss‚0.4508226907345823 /loss/FRt4/detection_loss‚2.6358165754212273 _runtime‚2497.525759644 .loss/FRt1/detection_loss‚3.062411743402481 1loss/SRqqt3/detection_loss‚1.2999367895810043 % _timestamp‚1.7884473820542653e+09 0loss/FBt4/assignment_loss‚0.2184259554131586–Éþ_ÆÙ 0loss/FRt2/assignment_loss‚4.9159549077351885 2loss/SRqqt2/assignment_loss‚1.8340198722978434 1loss/SRqqt4/detection_loss‚1.8032228521636275 .loss/FBt4/detection_loss‚3.387626016139984 _step‚257 /loss/FRt1/assignment_loss‚5.498709829648336 2loss/SRqqt3/assignment_loss‚1.2969095366733039 /loss/FRt3/detection_loss‚1.9365627033015091 0loss/FBt2/assignment_loss‚0.4529741532933258 1loss/FBt3/assignment_loss‚0.27332194256492065 /loss/FRt2/detection_loss‚2.0455879156003918 1loss/SRqqt1/detection_loss‚2.7019836008548737 %loss/total_loss‚52.55957688395345 epoch‚2 trainer/global_step‚3149 /loss/FRt3/assignment_loss‚2.476622256139914 0loss/FBt1/assignment_loss‚0.4508226907345823 /loss/FRt4/detection_loss‚2.6358165754212273 _runtime‚2497.525759644 .loss/FRt1/detection_loss‚3.062411743402481 1loss/SRqqt3/detection_loss‚1.2999367895810043 % _timestamp‚1.7884473820542653e+09 0loss/FBt4/assignment_loss‚0.2184259554131586 0loss/FRt4/assignment_loss‚3.9562379227744207 2loss/SRqqt1/assignment_loss‚2.5941291550795236 1loss/SRqqt2/detection_loss‚1.6939995139837265 /loss/FBt1/detection_loss‚3.4858792573213577 .loss/FBt2/detection_loss‚2.307929197947184 1loss/SRqqt4/assignment_loss‚1.558844307773426 /loss/FBt3/detection_loss‚0.6716481881081372 YféžÇ‚(:“ •æÔÈÚûü gpu.0.gpu‚47.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚132.228' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.445600000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0”ä¤:EÈ‚(:º •æÔü¤êÿ disk.drbd2080.in‚0$ disk.nvme0c0n1.out‚ 9717.90234375 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda1.in‚0 disk.dm-1.in‚0 network.sent‚ 6.650976e+06* disk./.usagePercent‚20.073385890929156$ disk./.usageGB‚299.1079559326172 disk.drbd1716.in‚0 disk.sda3.out‚ 95.42578125 disk.drbd2266.out‚0 disk.dm-1.out‚ 95.42578125 network.recv‚911758 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda1.out‚0 disk.rbd1.out‚ 13.345703125" disk.nvme0n1.out‚ 9717.90234375 disk.drbd2266.in‚0 disk.sda2.out‚0" proc.memory.rssMB‚ 2642.7109375 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125$ memory_percent‚95.54574489593506 disk.sda.out‚ 95.42578125 disk.rbd0.out‚0 disk.dm-0.out‚ 9717.90234375 disk.sda2.in‚0( proc.memory.availableMB‚ 729.78515625) proc.memory.percent‚16.12982749938965 cpu‚25.304666016962962µ¤IÉ‚(:’ ¬•æÔ×ËŒý gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚156.76' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚31.351999999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ýÄ'àEÊ‚(:º ¬•æÔëåê€ disk.drbd1716.out‚0 disk.nvme0n1.in‚0.078125" disk.nvme0n1.out‚ 9787.24609375$ disk./.usageGB‚299.1323051452637 disk.sda.in‚0 disk.dm-0.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0' proc.memory.availableMB‚ 730.0234375 proc.cpu.threads‚17 disk.sda3.out‚ 96.02734375) proc.memory.percent‚16.12982749938965 cpu‚25.303146658068307 network.sent‚ 6.759951e+06 disk.rbd1.out‚ 13.4638671875 disk.drbd2080.out‚0 disk.dm-1.out‚ 96.02734375 disk.drbd1716.in‚0* disk./.usagePercent‚20.075019987020212 disk.sda1.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 96.02734375$ disk.nvme0c0n1.out‚ 9787.24609375 disk.sda3.in‚0" proc.memory.rssMB‚ 2642.7109375 network.recv‚918920 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.dm-0.out‚ 9787.24609375$ memory_percent‚95.54429054260254 disk.sda2.out‚0*ù"Ü2Ë- _wandbruntime‚2533  _runtime‚25337ý ¿ÉÌà % _timestamp‚1.7884474181064339e+09 _runtime‚2533.578074266 _step‚258 ' lr-AdamW/pg1‚0.00011651488434602152 ' lr-AdamW/pg2‚0.00011651488434602152 trainer/global_step‚3199a(ä>ÎÍÈ _runtime‚2533.578074266 _step‚258 ' lr-AdamW/pg1‚0.00011651488434602152 ' lr-AdamW/pg2‚0.00011651488434602152 trainer/global_step‚3199 % _timestamp‚1.7884474181064339e+09‚¶`‰2Î- _wandbruntime‚2533  _runtime‚2533eØc(Y ÏÖO YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚184987202425344 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2668880598016 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚202075358257664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚187656757388288 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚182814973506533 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚2533.579094502 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2802370127872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚184987202425344 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚60185670 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2659713068381 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚187656083023360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46220352 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚187656757388288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26962804 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 202244608 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106426061 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46220354 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2669536960000 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚180155931915264 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚462 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚460 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚202075334317568 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42217632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚180155914875904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚187656083023360 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚184987220428288 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46220352 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46220354 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚204877728385536 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60205707 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚184987220428288 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 ZFDeviceStatsMonitor.on_train_batch_start/inactive_splÛH¾ƒit_bytes.all.freed‚204877526140928 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.788447418107165e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15254826 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2802191823360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60205707 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2668880598016 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26962806 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 178304512R¼Ü%á'ÐÛO M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42217170 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚182815644983645 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2659058630629 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚259 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚3199 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2669536960000 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15254366 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚202075358257664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚187656757388288 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚182814973506533 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚2533.579094502 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2802370127872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚184987202425344 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚60185670 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2659713068381 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚187656083023360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46220352 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚187656757388288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26962804 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 202244608 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106426061 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46220354 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2669536960000 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚180155931915264 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚462 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚460 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚202075334317568 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42217632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚180155914875904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚187656083023360 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚184987220428288 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46220352 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46220354 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚204877728385536 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60205707 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚184987220428288 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚204877526140928 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 $ _timestamp‚1.788447418107165e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15254826 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2802191823360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60205707 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2668880598016 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26962806 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 178304512 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚184987202425344 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2668880598016 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106458557 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60204124 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106438492 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15259304 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚182870694072057 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚260 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46234368 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2669700025856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚185043555464192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2670356401152 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 % _timestamp‚1.7884474190385413e+09 _runtime‚2534.510549028 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚187713929868288 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46234368 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60224187 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15258867 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚182871365549377 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42229839 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚187713255490048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189648384 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚185043573467136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60204124 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚204940033504256 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2660529503553 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46234370 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚202136985055232 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2803048449024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚180210836045824 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60224187 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2v6jcì&ÓæM F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚187713255490048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189648384 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚185043573467136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60204124 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚204940033504256 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2660529503553 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46234370 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚202136985055232 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2803048449024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚180210836045824 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60224187 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 trainer/global_step‚3199 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚185043573467136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2670356401152 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46234370 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚180210819006464 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42230278 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚202137008995328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2659875065593 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2803214157312 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165708288 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106458557 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60204124 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106438492 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15259304 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚182870694072057 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚260 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46234368 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2669700025856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚185043555464192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2670356401152 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 % _timestamp‚1.7884474190385413e+09 _runtime‚2534.510549028 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚187713929868288 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46234368 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60224187 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15258867 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚182871365549377 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42229839„‚º0€2Ô- _wandbruntime‚2534  _runtime‚2534@]nš4ÕÒ 0loss/FRt3/assignment_loss‚2.4853552658602878 1loss/SRqqt3/detection_loss‚1.5463270539683955 /loss/FRt2/assignment_loss‚5.019702914782933 1loss/FBt3/assignment_loss‚0.23057456364912793 0loss/FBt4/assignment_loss‚0.7324787061661482 epoch‚2 _step‚261 2loss/SRqqt2/assignment_loss‚1.9352340416775808 0loss/SRqqt1/detection_loss‚2.675279275729106 .loss/FBt4/detection_loss‚2.953073811531067 trainer/global_step‚3199 % _timestamp‚1.7884474190401945e+09 _runtime‚2534.511698158 0loss/FBt1/assignment_loss‚Mø¹^$0.5512002135316152 .loss/FRt4/detection_loss‚2.323584715322572 .loss/FBt2/detection_loss‚2.558307483792305 &loss/total_loss‚52.089453114792015 2loss/SRqqt1/assignment_loss‚2.8235393304091234 2loss/SRqqt3/assignment_loss‚1.4419499522417176 .loss/FRt3/detection_loss‚1.830369569220633 /loss/FRt1/detection_loss‚3.1186016499996185 .loss/FBt1/detection_loss‚3.237779849105411 .loss/FBt3/detection_loss‚0.689189012098888 0loss/FBt2/assignment_loss‚0.5576810101978481 .loss/FRt2/detection_loss‚2.145531952381134 0loss/FRt1/assignment_loss‚4.7446107523781915 0loss/FRt4/assignment_loss‚3.0478601966585432 2loss/SRqqt4/assignment_loss‚1.8522766850522308 1loss/SRqqt2/detection_loss‚1.8834272490607367 1loss/SRqqt4/detection_loss‚1.7055178599767997©ÅÏ]Ö× 0loss/FBt2/assignment_loss‚0.5576810101978481 .loss/FRt2/detection_loss‚2.145531952381134 0loss/FRt1/assignment_loss‚4.7446107523781915 0loss/FRt4/assignment_loss‚3.0478601966585432 2loss/SRqqt4/assignment_loss‚1.8522766850522308 1loss/SRqqt2/detection_loss‚1.8834272490607367 1loss/SRqqt4/detection_loss‚1.7055178599767997 0loss/FRt3/assignment_loss‚2.4853552658602878 1loss/SRqqt3/detection_loss‚1.5463270539683955 /loss/FRt2/assignment_loss‚5.019702914782933 1loss/FBt3/assignment_loss‚0.23057456364912793 0loss/FBt4/assignment_loss‚0.7324787061661482 epoch‚2 _step‚261 2loss/SRqqt2/assignment_loss‚1.9352340416775808 0loss/SRqqt1/detection_loss‚2.675279275729106 .loss/FBt4/detection_loss‚2.953073811531067 trainer/global_step‚3199 % _timestamp‚1.7884474190401945e+09 _runtime‚2534.511698158 0loss/FBt1/assignment_loss‚0.5512002135316152 .loss/FRt4/detection_loss‚2.323584715322572 .loss/FBt2/detection_loss‚2.558307483792305 &loss/total_loss‚52.089453114792015 2loss/SRqqt1/assignment_loss‚2.8235393304091234 2loss/SRqqt3/assignment_loss‚1.4419499522417176 .loss/FRt3/detection_loss‚1.830369569220633 /loss/FRt1/detection_loss‚3.1186016499996185 .loss/FBt1/detection_loss‚3.237779849105411 .loss/FBt3/detection_loss‚0.689189012098888…Ýë¿ “ׂ(:ˆ »•æÔû¤„ý gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.151' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4302 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¾p~:AØ‚(:¶ »•æÔåþšÿ disk.drbd2265.in‚0 disk.drbd2266.out‚0 disk.drbd2080.in‚0 disk.dm-0.out‚ 9813.05078125 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0) proc.memory.percent‚16.12982749938965* disk./.usagePercent‚20.075399134178422 disk.sda.in‚0" disk.nvme0n1.out‚ 9813.05078125 disk.drbd1716.out‚0 network.recv‚921998 disk.drbd2265.out‚0" disk.nvme0c0n1.out‚ 9812.953125 disk.sda3.in‚0 disk.rbd0.in‚0 disk.sda.out‚ 96.7421875 disk.sda1.in‚0 disk.rbd1.out‚ 13.5888671875% disk./.usageGB‚299.13795471191406 disk.sda3.out‚ 96.7421875 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0( proc.memory.availableMB‚ 729.76953125 proc.cpu.threads‚17 network.sent‚ 6.765062e+06 disk.dm-0.in‚0.078125 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 disk.dm-1.in‚0 disk.dm-1.out‚ 96.7421875# memory_percent‚95.5458402633667" proc.memory.rssMB‚ 2642.7109375 cpu‚25.301601598917866´4.K“Ù‚(:ˆ Ê•æÔåéŠý gpu.0.gpu‚30.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚164.678' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚32.9356 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¯>KÚ‚(:À Ê•æÔøª˜ÿ disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 97.4453125 disk.drbd2080.in‚0 disk.sda3.in‚0 network.recv‚929287 disk.sda.out‚ 97.4453125 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚9852.19970703125* disk./.usagePercent‚20.076641282734556% disk.nvme0n1.out‚9852.19970703125 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0$ memory_percent‚95.54619789123535' proc.memory.availableMB‚ 729.7109375 cpu‚25.300138774806484 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 13.6513671875 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0" proc.memory.rssMB‚ 2642.7109375 network.sent‚ 6.874518e+06 disk.sda1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.sda3.out‚ 97.4453125" disk.dm-0.out‚9852.19970703125 disk.sda.in‚0 disk.sda2.out‚0 disk.dm-1.in‚0 disk.drbd2266.out‚0) proc.memory.percent‚16.12982749938965 proc.cpu.threads‚17$ disk./.usageGB‚299.1564636230469 disk.sda2.in‚0ñHl¨“Û‚(:ˆ Ù•æÔ¤ý gpu.0.gpu‚45.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.949' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5898 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0±zËAJÜ‚(:¿ Ù•æÔŠš»ÿ disk.rbd1.out‚ 13.6513671875 disk.sda3.out‚ 97.83984375 disk.drbd2266.in‚0 disk.rbd0.in‚0 network.recv‚931972& disk.nvme0c0n1.out‚9896.0439453125 cpu‚25.298687186097442 disk.sda3.in‚0 network.sent‚ 6.879443e+06 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.dm-1.out‚ 97.83984375" proc.memory.rssMB‚ 2642.7109375$ disk./.usageGB‚299.1733932495117 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0! disk.dm-0.out‚9896.0439453125 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.drbd2080.out‚0' proc.memory.availableMB‚ 729.9609375* disk./.usagePercent‚20.077777444171517 disk.sda2.in‚0$ disk.nvme0n1.out‚9896.0439453125 disk.drbd2266.out‚0# memory_percent‚95.5446720123291 disk.sda.in‚0 disk.sda.out‚ 97.83984375 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0) proc.memory.percent‚16.12982749938965pš‘2Ý- _wandbruntime‚2570  _runtime‚2570?øÝÉÞà ' lr-AdamW/pg1‚0.00011601240332485755 ' lr-AdamW/pg2‚0.00011601240332485755 trainer/global_step‚3249 % _timestamp‚1.7884474550619166e+09 _runtime‚2570.533515361 _step‚262Û×iYÎßÈ ' lr-AdamW/pg2‚0.00011601240332485755 trainer/global_step‚3249 % _timestamp‚1.7884474550619166e+09 _runtime‚2570.533515361 _step‚262 ' lr-AdamW/pg1‚0.00011601240332485755†4€g2à- _wandbruntime‚2570  _runtime‚2570™WïIÝ'á×O D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚190514720771584 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚187804872370688 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46921152 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚182901138443264 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61111722 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚185601686851053 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚447 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2709866403840 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15481207 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚208003391057920 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2700548407789 trainer/global_step‚3249 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46921154 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46921154 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27371204 _runtime‚2570.534495301 _step‚263 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚187804854367744 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181273088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚205157871200768 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚208003209784832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61131759 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2845338584064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2699893970037 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 108052913 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42852411 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332992k~YŽâ'âÜO KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2699893970037 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 108052913 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42852411 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332992 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚190514720771584 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚187804872370688 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46921152 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚182901138443264 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚61111722 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚185601686851053 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚447 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2709866403840 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15481207 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚208003391057920 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2700548407789 trainer/global_step‚3249 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46921154 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚46921154 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27371204 _runtime‚2570.534495301 _step‚263 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚187804854367744 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181273088 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚205157871200768 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚208003209784832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚61131759 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2845338584064‡üžÅ2ã- _wandbruntime‚2571  _runtime‚2571±cƒ°”äàM WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46935168 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153125376 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108085409 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚205219521938432 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚185656735939465 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2701364842961 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27379372 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚190571893238272 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61150239 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚190572567616512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46935170 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61130176 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚2571.217866169 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚187861225409536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚182956042573824 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42865572 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177065472 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185657407416785 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚3249 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42865119 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_po³:¶Rol.allocated‚187861225409536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15486198 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15485747 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 108065344 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61130176 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚208065913622528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2846367744000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2711342206976 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚208065736557056 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2700710405001 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚187861207406592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 108065344 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2710685831680 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚264 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚187861207406592 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2710685831680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61150239 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 $ _timestamp‚1.788447455745868e+09 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928æÈë&ååM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2710685831680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚61150239 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 $ _timestamp‚1.788447455745868e+09 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46935168 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153125376 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 108085409 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚205219521938432 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚185656735939465 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2701364842961 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27379372 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚190571893238272 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚61150239 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚190572567616512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46935170 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚61130176 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚2571.217866169 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚187861225409536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚182956042573824 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42865572 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177065472 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185657407416785 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚3249 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42865119 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚187861225409536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15486198 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15485747 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 108065344 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚61130176 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚208065913622528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2846367744000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2711342206976 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚208065736557056 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2700710405001 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚187861207406592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 108065344 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2710685831680 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚264 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚187861207406592 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424ˆ e52æ- _wandbruntime‚2571  _runtime‚25716lK^UçÏ _step‚265 0loss/FRt1/assignment_loss‚5.0393648246924085 0loss/FBt2/assignment_loss‚0.5266761481758473 /loss/FBt1/detection_loss‚3.1145230968793234 #loss/total_loss‚54.683326829815 0loss/FBt3/assignment_loss‚0.2771494748010792 /loss/FBt4/assignment_loss‚0.179537832736969 /loss/FRt1/detection_loss‚3.2760383412241936 0loss/FRt4/assignment_loss‚3.9436318198839824 1loss/SRqqt2/detection_loss‚1.9229451076550916 1loss/SRqqt4/detection_loss‚1.9532825310351485 .loss/FBt2/detection_loss‚2.286401262650123 /loss/FBt4/detection_loss‚6.4221581518650055 epoch‚2 /loss/FRt2/detection_loss‚2.4458543054367365 0loss/FRt2/assignment_loss‚3.9660850512353996 2loss/SRqqt3/assignment_loss‚1.2718582165372723 2loss/SRqqt4/assignment_loss‚1.8262770210029715 0loss/SRqqt1/detection_loss‚2.732841425820401 1loss/SRqqt3/detection_loss‚1.2784439259592224 /loss/FRt3/assignment_loss‚2.244207228422165 0loss/FBt1/assignment_loss‚0.6308106463605554 -loss/FRt3/detection_loss‚1.89877225369215 trainer/global_step‚3249 2loss/SRqqt1/assignment_loss‚2.1838758878017726 2loss/SRqqt2/assignment_loss‚1.9519066207788207 -loss/FRt4/detection_loss‚2.54249796933598 /loss/FBt3/detection_loss‚0.7681876858323813 % _timestamp‚1.7884474557478864e+09 _runtime‚2571.219396247Š?õÌZèÔ 1loss/SRqqt2/detection_loss‚1.9229451076550916 1loss/SRqqt4/detection_loss‚1.9532825310351485 .loss/FBt2/detection_loss‚2.286401262650123 /loss/FBt4/detection_loss‚6.4221581518650055 epoch‚2 /loss/FRt2/detection_loss‚2.4458543054367365 0loss/FRt2/assignment_loss‚3.9660850512353996 2loss/SRqqt3/assignment_loss‚1.2718582165372723 2loss/SRqqt4/assignment_loss‚1.8262770210029715 0loss/SRqqt1/detection_loss‚2.732841425820401 1loss/SRqqt3/detection_loss‚1.2784439259592224 /loss/FRt3/assignment_loss‚2.244207228422165 0loss/FBt1/assignment_loss‚0.6308106463605554 -loss/FRt3/detection_loss‚1.89877225369215 trainer/global_step‚3249 2loss/SRqqt1/assignment_loss‚2.1838758878017726 2loss/SRqqt2/assignment_loss‚1.9519066207788207 -loss/FRt4/detection_loss‚2.54249796933598 /loss/FBt3/detection_loss‚0.7681876858323813 % _timestamp‚1.7884474557478864e+09 _runtime‚2571.219396247 _step‚265 0loss/FRt1/assignment_loss‚5.0393648246924085 0loss/FBt2/assignment_loss‚0.5266761481758473 /loss/FBt1/detection_loss‚3.1145230968793234 #loss/total_loss‚54.683326829815 0loss/FBt3/assignment_loss‚0.2771494748010792 /loss/FBt4/assignment_loss‚0.179537832736969 /loss/FRt1/detection_loss‚3.2760383412241936 0loss/FRt4/assignment_loss‚3.9436318198839824‰ ŠûÍžé‚(:“ è•æÔ¹²€þ gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.737' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.547399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0h­ÐJê‚(:¿ è•æÔ ‘¯€ disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 cpu‚25.29714364330741 disk.sda3.in‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 network.sent‚ 6.885084e+06* disk./.usagePercent‚20.078171183759178 disk.drbd2080.out‚0 disk.sda2.in‚0% disk.nvme0n1.out‚9942.32958984375 disk.drbd1716.out‚0 proc.cpu.threads‚17 network.recv‚936580 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚9942.32958984375 disk.drbd2266.out‚0 disk.dm-1.in‚0# memory_percent‚95.5449104309082 disk.drbd2080.in‚0 disk.sda3.out‚ 98.51953125 disk.rbd0.out‚0 disk.drbd2265.out‚0& proc.memory.availableMB‚ 729.921875" proc.memory.rssMB‚ 2642.7109375 disk.drbd2266.in‚0 disk.sda.in‚0 disk.sda.out‚ 98.51953125 disk.sda2.out‚0) proc.memory.percent‚16.12982749938965% disk./.usageGB‚299.17926025390625" disk.dm-0.out‚9942.32958984375 disk.dm-1.out‚ 98.51953125 disk.rbd1.in‚0 disk.rbd1.out‚ 13.68359375 disk.drbd1716.in‚0eÆ-òžë‚(:“ ÷•æÔÇ„íü gpu.0.gpu‚56.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.296' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.459199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0eôXIì‚(:¾ ÷•æÔÃÃç€ disk.nvme0n1.in‚0.078125* disk./.usagePercent‚20.079332689942024 disk.drbd2080.in‚0% disk.nvme0n1.out‚9996.07568359375 disk.drbd2265.in‚0 disk.rbd0.in‚0" disk.dm-0.out‚9996.07568359375 disk.sda.out‚ 98.8203125$ memory_percent‚95.54495811462402 disk.sda3.out‚ 98.8203125 disk.drbd2266.out‚0 network.sent‚ 6.992969e+06$ disk./.usageGB‚299.1965675354004 disk.drbd1716.in‚0 cpu‚25.295721753446486 disk.drbd2265.out‚0 disk.sda1.in‚0 disk.dm-1.out‚ 98.8203125" proc.memory.rssMB‚ 2642.7109375 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚9996.07568359375 disk.sda3.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd1716.out‚0) proc.memory.percent‚16.12982749938965 disk.rbd1.out‚ 13.80859375 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.out‚0 disk.dm-1.in‚0' proc.memory.availableMB‚ 729.9140625 network.recv‚941789 disk.dm-0.in‚0.078125 disk.rbd1.in‚0D“2í- _wandbruntime‚2607  _runtime‚2607ÂYg’Éîà % _timestamp‚1.7884474920237029e+09 _runtime‚2607.495284728 _step‚266 ' lr-AdamW/pg1‚0.00011549894604696878 ' lr-AdamW/pg2‚0.00011549894604696878 trainer/global_step‚3299ŒŽ°ÇÎïÈ _runtime‚2607.495284728 _step‚266 ' lr-AdamW/pg1‚0.00011549894604696878 ' lr-AdamW/pg2‚0.00011549894604696878 trainer/global_step‚3299 % _timestamp‚1.7884474920237029e+09Šÿl`g2ð- _wandbruntime‚2607  _runtime‚26078B“jÝ'ñ×O RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2888231431168 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47621952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚211128818691072 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚188387057241349 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47621952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚185646344971264 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚3299 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚190622506310144 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43488060 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2740729309445 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2750852209664 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚193373358519808 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2750852209664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚208240408083968 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚185646327931904 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2741383747197 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2751508571648 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚431 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚62037774 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 109659726 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚208240432024064 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚190622524313088 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2751508571648 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62057811 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2888386667008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚2607.496187141 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 109679765 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚429 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚193373358519808 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚193374032884736 % _timestamp‚1.7884474920243728e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844Ã'ÒòÜO HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2750852209664 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚193373358519808 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2750852209664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚208240408083968 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚185646327931904 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2741383747197 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2751508571648 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚431 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚62037774 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 109659726 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚208240432024064 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚190622524313088 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2751508571648 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62057811 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2888386667008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚2607.496187141 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 109679765 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚429 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚193373358519808 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚193374032884736 % _timestamp‚1.7884474920243728e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2888231431168 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47621952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚211128818691072 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚188387057241349 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47621952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚185646344971264 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚3299 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚190622506310144 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43488060 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2740729309445 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844‹;vÙÁ2ó- _wandbruntime‚2608  _runtime‚2608h‡{5ç&ôáM D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 109692196 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚62056228 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2889093027328 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚193431205364736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚193431205364736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47635968 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47635968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _step‚268 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15713386 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2889254541312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43500715 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 109692196 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513984 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62056228 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚188442777806873 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47635970 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884474927181213e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109712261 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185454080 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚208302082761728 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚190678859348992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62076291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43501160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚211191337303040 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚211191151848960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚190678859348992 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2741545744409 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15712943 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚185701249101824 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚188443449284193 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2742200182369 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 trainer/global_step‚3299 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2751671637504 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109712261 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0‰¤ :ì&õæM JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 trainer/global_step‚3299 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2751671637504 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 109712261 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 109692196 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚62056228 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2889093027328 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚193431205364736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚193431205364736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47635968 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47635968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _step‚268 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚15713386 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2889254541312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43500715 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 109692196 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513984 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚62056228 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚188442777806873 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47635970 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884474927181213e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 109712261 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185454080 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚208302082761728 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚190678859348992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚62076291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43501160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚211191337303040 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚211191151848960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚190678859348992 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2741545744409 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15712943 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚185701249101824 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚188443449284193 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2742200182369Œ§ïr2ö- _wandbruntime‚2608  _runtime‚26082¤DP÷Ê /loss/FRt3/assignment_loss‚2.144803444976392 1loss/SRqqt2/assignment_loss‚1.832353394579243 1loss/FBt1/assignment_loss‚0.41364291159303407 0loss/FBt4/assignment_loss‚0.5752393116043615 /loss/FRt3/detection_loss‚1.9926056988213374 1loss/SRqqt3/assignment_loss‚1.183758662827313 1loss/SRqqt4/detection_loss‚2.0757844277002193 .loss/FBt1/detection_loss‚3.281904971599579 /loss/FRt4/assignment_loss‚3.304078135978092 1loss/SRqqt1/assignment_loss‚2.125197806141593 /loss/FRt2/detection_loss‚2.2193197267396108 0loss/SRqqt1/detection_loss‚2.533189131454988 %loss/total_loss‚50.50921300122556 trainer/global_step‚3299 % _timestamp‚1.7884474927200186e+09 /loss/FRt2/assignment_loss‚4.355627954006195 2loss/SRqqt4/assignment_loss‚1.9226532098319795 0loss/FBt3/assignment_loss‚0.2655676351706086 0loss/SRqqt3/detection_loss‚1.236319072331701 _step‚269 /loss/FBt3/detection_loss‚0.7679213850324352 epoch‚2 _runtime‚2608.191522533 .loss/FRt1/detection_loss‚3.251213952898979 0loss/SRqqt2/detection_loss‚2.148024232000918 0loss/FBt2/assignment_loss‚0.6415053517488327 .loss/FRt1/assignment_loss‚4.63236462076505 .loss/FRt4/detection_loss‚2.345352598211982 .loss/FBt2/detection_loss‚2.171789478510618 .loss/FBt4/detection_loss‚3.088995886700494K$ƒiUøÏ .loss/FRt1/detection_loss‚3.251213952898979 0loss/SRqqt2/detection_loss‚2.148024232000918 0loss/FBt2/assignment_loss‚0.6415053517488327 .loss/FRt1/assignment_loss‚4.63236462076505 .loss/FRt4/detection_loss‚2.345352598211982 .loss/FBt2/detection_loss‚2.171789478510618 .loss/FBt4/detection_loss‚3.088995886700494 /loss/FRt3/assignment_loss‚2.144803444976392 1loss/SRqqt2/assignment_loss‚1.832353394579243 1loss/FBt1/assignment_loss‚0.41364291159303407 0loss/FBt4/assignment_loss‚0.5752393116043615 /loss/FRt3/detection_loss‚1.9926056988213374 1loss/SRqqt3/assignment_loss‚1.183758662827313 1loss/SRqqt4/detection_loss‚2.0757844277002193 .loss/FBt1/detection_loss‚3.281904971599579 /loss/FRt4/assignment_loss‚3.304078135978092 1loss/SRqqt1/assignment_loss‚2.125197806141593 /loss/FRt2/detection_loss‚2.2193197267396108 0loss/SRqqt1/detection_loss‚2.533189131454988 %loss/total_loss‚50.50921300122556 trainer/global_step‚3299 % _timestamp‚1.7884474927200186e+09 /loss/FRt2/assignment_loss‚4.355627954006195 2loss/SRqqt4/assignment_loss‚1.9226532098319795 0loss/FBt3/assignment_loss‚0.2655676351706086 0loss/SRqqt3/detection_loss‚1.236319072331701 _step‚269 /loss/FBt3/detection_loss‚0.7679213850324352 epoch‚2 _runtime‚2608.191522533þWèlžù‚(:“ †–æÔŽ§©ý gpu.0.gpu‚46.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚151.791' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚30.358200000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¢šÄ)7ú‚(:¾ †–æÔ³ÜÈÿ disk.sda2.out‚0 disk.rbd1.out‚ 13.87109375 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.sda.in‚0 disk.rbd1.in‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0# disk.dm-0.out‚10041.25439453125* disk./.usagePercent‚20.080413809759122$ disk./.usageGB‚299.2126770019531 network.sent‚ 6.999571e+06 disk.sda1.out‚0 disk.drbd2265.out‚0 cpu‚25.29444448936825 network.recv‚946911 disk.sda2.in‚0 disk.dm-1.out‚©#›5 99.453125 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.dm-0.in‚0.078125( proc.memory.availableMB‚ 729.87890625) proc.memory.percent‚16.12982749938965 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚10041.25439453125( disk.nvme0c0n1.out‚10041.25439453125 disk.sda3.out‚ 99.453125" proc.memory.rssMB‚ 2642.7109375 disk.dm-1.in‚0 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.sda.out‚ 99.453125$ memory_percent‚95.54517269134521 proc.cpu.threads‚176ÞÛ“û‚(:ˆ •–æÔ¡øü gpu.0.gpu‚55.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.297' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.6594 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0o*Y(Kü‚(:À •–æÔмòÿ# disk.dm-0.out‚10080.54345703125 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 99.9296875 disk.drbd2265.in‚0 proc.cpu.threads‚17 network.sent‚ 7.107319e+06 disk.nvme0c0n1.in‚0.078125* disk./.usagePercent‚20.081793434361142 disk.sda.in‚0& disk.nvme0n1.out‚10080.54345703125 disk.drbd1716.in‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0" proc.memory.rssMB‚ 2642.7109375 disk.dm-1.in‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 network.recv‚952349 disk.dm-0.in‚0.078125) proc.memory.percent‚16.12982749938965 disk.sda.out‚ 99.9296875 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚10080.54345703125 disk.sda3.out‚ 99.9296875& proc.memory.availableMB‚ 729.828125 cpu‚25.293056831064224 disk.sda2.in‚0$ disk./.usageGB‚299.2332344055176 disk.rbd0.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 13.87109375$ memory_percent‚95.54548263549805ù¾’ý‚(:‡ ¤–æÔîªïü gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.699' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.3398 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“ÈâƒLþ‚(:Á ¤–æÔ’寀) disk./.usagePercent‚20.08379771734671 disk.sda.out‚ 100.40625 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚10127.02197265625( disk.nvme0c0n1.out‚10127.02197265625 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0$ memory_percent‚95.54550647735596 network.sent‚ 7.113716e+06 disk.drbd2266.out‚0) proc.memory.percent‚16.12982749938965 cpu‚25.291681448046912 disk.rbd0.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 100.40625 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2265.out‚0( proc.memory.availableMB‚ 729.82421875" proc.memory.rssMB‚ 2642.7109375 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 network.recv‚957139 disk.sda3.out‚ 100.40625 disk.rbd1.out‚ 13.9033203125 proc.cpu.threads‚17% disk./.usageGB‚299.26309967041016 disk.nvme0c0n1.in‚0.078125# disk.dm-0.out‚10127.02197265625 disk.sda.in‚0ÕØœ2ÿ- _wandbruntime‚2644  _runtime‚2644ÿ"eȀ ' lr-AdamW/pg1‚0.00011497461951608232 ' lr-AdamW/pg2‚0.00011497461951608232 trainer/global_step‚3349 % _timestamp‚1.7884475286386583e+09 _runtime‚ 2644.11024749 _step‚270 *ÔÍÇ trainer/global_step‚3349 % _timestamp‚1.7884475286386583e+09 _runtime‚ 2644.11024749 _step‚270 ' lr-AdamW/pg1‚0.00011497461951608232 ' lr-AdamW/pg2‚0.00011497461951608232Žzø¢2‚- _wandbruntime‚2644  _runtime‚2644ø[1Ý'ƒ×O UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62963826 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚196232670632960 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62983863 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2782219086605 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2781564648853 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183370240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚196232670632960 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚196231996268032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48322752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111306617 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44125494 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚193440176255488 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2792494377472 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚211322944967168 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 111286578 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚196231996268032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2792494377472 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28188004 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15937046 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159430144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2931310018560 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚193440158252544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2791838015488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48322752 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚211322968907264 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚191173099108757 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚214254438355968 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15937488 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚193440158252544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2791838015488 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 % _timestamp‚1.7884475286393554e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚191173770585869 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430XóýÔâ'„ÜO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚191173770585869 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚62963826 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚196232670632960 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚62983863 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2782219086605 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2781564648853 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183370240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚196232670632960 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚196231996268032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚48322752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 111306617 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44125494 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚193440176255488 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2792494377472 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚211322944967168 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 111286578 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚196231996268032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2792494377472 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28188004 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚15937046 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159430144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2931310018560 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚193440158252544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2791838015488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚48322752 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚211322968907264 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚191173099108757 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚214254438355968 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚15937488 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚193440158252544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2791838015488 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 % _timestamp‚1.7884475286393554e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2793313818624 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚214316955759616 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _step‚272 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚193496511291392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48336770 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48336768 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚3349 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 $ _timestamp‚1.788447529324319e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2792657443328 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259776 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2792657443328 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚214316774499840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚211384595704832 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44137788 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚193496511291392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2932336114688 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚191229491151601 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _runtime‚ 2644.79642943 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63002343 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15941616 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2932178795008 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚188446455629824 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111339113 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚196289843112960 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111339113 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚211384619644928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414bÿê&‡äM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚63002343 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚193496529294336 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 111319048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚48336770 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚48336768 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 trainer/global_step‚3349 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 $ _timestamp‚1.788447529324319e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2792657443328 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259776 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2792657443328 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚214316774499840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚211384595704832 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44137788 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚193496511291392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2932336114688 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚191229491151601 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _runtime‚ 2644.79642943 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63002343 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚15941616 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2932178795008 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚188446455629824 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 111339113 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚196289843112960 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 111339113 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚211384619644928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 111319048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚193496529294336 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44138233 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2793313818624 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚214316955759616 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _step‚272 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚193496511291392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚199090634016256 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚217380037097984 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2832823821312 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63889878 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2833480183296 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 $ _timestamp‚1.788447565132805e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2832823821312 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49023554 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚196257828197888 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚199090634016256 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚44761801 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚191136740987904 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚450 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚196257810194944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49023552 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚193959140976165 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16165845 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚199091308381184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2974531307520 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039®-*”ÛO _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2974367683072 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚63909915 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28596406 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚217379849533440 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚214405481850368 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚199090634016256 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚217380037097984 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2832823821312 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚63889878 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_byteS_Ÿš·s.small_pool.allocated‚ 2833480183296 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 $ _timestamp‚1.788447565132805e+09 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2832823821312 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49023554 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚196257828197888 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚199090634016256 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚44761801 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚191136740987904 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚450 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚196257810194944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49023552 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚193959140976165 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16165845 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚199091308381184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2974531307520 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564544 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 112913430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚196257810194944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28596404 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2823054426013 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44762251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚49023554 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163624448 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚2680.604630414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 112933469 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚63909915 trainer/global_step‚3399 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚199091308381184 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2833480183296“­gsà2•- _wandbruntime‚2681  _runtime‚2681U3íç&–áM I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63908332 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 % _timestamp‚1.7884475658236618e+09 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2975241125888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚196314181236736 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚196314163233792 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63928395 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚2681.295687649 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28604574 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚191191662157824 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49037570 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚432 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49037570 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44774486 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16170344 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 trainer/global_step‚3399 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚217442552876544 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2823216423225 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚63928395 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63908332 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 112965965 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28604572 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚214467156528128 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚194015533019009 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 112945900 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2833643249152 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112965965 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16169914 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚196314163233792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112945900 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚217442373713920 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49037568 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2823870861185 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44774918 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2833643249152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49037568 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2834299624448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _step‚276 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0$tpŸì&—æM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28604574 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚191191662157824 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49037570 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚432 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49037570 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44774486 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16170344 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 trainer/global_step‚3399 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚217442552876544 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2823216423225 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚63928395 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚63908332 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 112965965 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28604572 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚214467156528128 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚194015533019009 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 112945900 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2833643249152 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 112965965 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16169914 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚196314163233792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 112945900 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚217442373713920 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49037568 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2823870861185 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44774918 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2833643249152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49037568 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2834299624448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _step‚276 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚63908332 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 % _timestamp‚1.7884475658236618e+09 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2975241125888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚196314181236736 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚196314163233792 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚63928395 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚2681.295687649 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844”/“?ž2˜- _wandbruntime‚2681  _runtime‚2681È~×U™Ï /loss/FBt2/detection_loss‚2.4866128976528463 $ _timestamp‚1.788447565825304e+09 /loss/FRt4/assignment_loss‚3.436718201637268 0loss/FBt2/assignment_loss‚0.8564581627454118 2loss/SRqqt2/assignment_loss‚1.9725409815708796 0loss/SRqqt3/detection_loss‚1.286832289003274 /loss/FRt4/detection_loss‚2.0545546661723746 0loss/SRqqt1/detection_loss‚2.469187160332998 0loss/SRqqt2/detection_loss‚1.876412096950743 /loss/FBt1/detection_loss‚2.9465232342481613 _step‚277 1loss/SRqqt1/assignment_loss‚2.539943811794122 /loss/FRt1/detection_loss‚3.3297078895568846 /loss/FRt3/detection_loss‚1.8003015407809504 /loss/FBt4/detection_loss‚3.6325763940811155 epoch‚2 _runtime‚2681.296807042 /loss/FRt1/assignment_loss‚4.719852247238159 .loss/FRt3/assignment_loss‚2.13039994681323 2loss/SRqqt4/assignment_loss‚1.7821595452725887 0loss/FBt3/assignment_loss‚0.2640883075050143 /loss/FBt3/detection_loss‚0.7119660273288624 %loss/total_loss‚51.52649564935499 trainer/global_step‚3399 /loss/FRt2/detection_loss‚2.6397019434306355 1loss/SRqqt4/detection_loss‚1.8367070420086384 /loss/FRt2/assignment_loss‚4.252124418814977 0loss/FBt1/assignment_loss‚0.4240435883402771 0loss/FBt4/assignment_loss‚0.5022234828558567 2loss/SRqqt3/assignment_loss‚1.5748597732197274Üd2éZšÔ /loss/FRt1/assignment_loss‚4.719852247238159 .loss/FRt3/assignment_loss‚2.13039994681323 2loss/SRqqt4/assignment_loss‚1.7821595452725887 0loss/FBt3/assignment_loss‚0.2640883075050143 /loss/FBt3/detection_loss‚0.7119660273288624 %loss/total_loss‚51.52649564935499 trainer/global_step‚3399 /loss/FRt2/detection_loss‚2.6397019434306355 1loss/SRqqt4/detection_loss‚1.8367070420086384 /loss/FRt2/assignment_loss‚4.252124418814977 0loss/FBt1/assignment_loss‚0.4240435883402771 0loss/FBt4/assignment_loss‚0.5022234828558567 2loss/SRqqt3/assignment_loss‚1.5748597732197274 /loss/FBt2/detection_loss‚2.4866128976528463 $ _timestamp‚1.788447565825304e+09 /loss/FRt4/assignment_loss‚3.436718201637268 0loss/FBt2/assignment_loss‚0.8564581627454118 2loss/SRqqt2/assignment_loss‚1.9725409815708796 0loss/SRqqt3/detection_loss‚1.286832289003274 /loss/FRt4/detection_loss‚2.0545546661723746 0loss/SRqqt1/detection_loss‚2.469187160332998 0loss/SRqqt2/detection_loss‚1.876412096950743 /loss/FBt1/detection_loss‚2.9465232342481613 _step‚277 1loss/SRqqt1/assignment_loss‚2.539943811794122 /loss/FRt1/detection_loss‚3.3297078895568846 /loss/FRt3/detection_loss‚1.8003015407809504 /loss/FBt4/detection_loss‚3.6325763940811155 epoch‚2 _runtime‚2681.296807042•pî“›‚(:ˆ Ñ–æÔµð¡ý gpu.0.gpu‚40.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚173.846' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.7692 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¬èRëBœ‚(:· Ñ–æÔÿ¨€ network.recv‚971044 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 14.0908203125 disk.nvme0n1.in‚0.078125# disk.nvme0n1.out‚10255.06640625 disk.dm-1.out‚ 102.28125 disk.drbd1716.out‚0 disk.sda3.out‚ 102.28125 disk.drbd2266.out‚0$ memory_percent‚95.54829597473145 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 102.28125 disk.sda1.out‚0 cpu‚25.28758364517778 proc.cpu.threads‚17 network.sent‚ 7.233272e+06* disk./.usagePercent‚20.087444800679496 disk.sda2.in‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125" proc.memory.rssMB‚ 2642.7109375% disk./.usageGB‚299.31744384765625 disk.rbd1.in‚0% disk.nvme0c0n1.out‚10255.06640625 disk.sda1.in‚0) proc.memory.percent‚16.12982749938965 disk.dm-1.in‚0 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.dm-0.out‚10255.06640625' proc.memory.availableMB‚ 729.3671875îóËž‚(:“ à–æÔ‚Шý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.374' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.474799999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ï|Ê Hž‚(:½ à–æÔ¨ä›ÿ disk.drbd2266.in‚0 proc.cpu.threads‚17 disk.dm-1.out‚102.9375 disk.drbd1716.out‚0 network.sent‚ 7.342396e+06* disk./.usagePercent‚20.088811112889736 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda2.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚102.9375 cpu‚25.286207197410867 disk.drbd1716.in‚0 network.recv‚977932 disk.sda2.in‚0 disk.rbd1.out‚ 14.123046875 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚10298.23388671875 disk.rbd0.out‚0 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚10298.23388671875# disk.dm-0.out‚10298.23388671875 disk.sda1.out‚0$ memory_percent‚95.54831981658936$ disk./.usageGB‚299.3378028869629 disk.sda.out‚102.9375) proc.memory.percent‚16.12982749938965 disk.dm-1.in‚0 disk.rbd0.in‚0" proc.memory.rssMB‚ 2642.7109375 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125( proc.memory.availableMB‚ 729.36328125¹/Ÿ‚(:’ ï–æÔÀ¢Šý gpu.0.gpu‚35.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚172.531' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚34.50620000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ye"¿° ‚(:Ç ï–æÔÄÒïþ disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.sda.in‚0 disk.sda.out‚ 103.4453125 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 network.recv‚981319 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 14.333984375( disk.nvme0c0n1.out‚10346.45751953125 disk.sda3.in‚0 disk.drbd1716.out‚0 disk.sda1.in‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.dm-1.out‚ 103.4453125" proc.memory.rssMB‚ 2642.7109375 proc.cpu.threads‚17# disk.dm-0.out‚10346.45751953125 disk.sda1.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 103.4453125( proc.memory.availableMB‚ 7297I¾ ¢.36328125* disk./.usagePercent‚20.091749311360214& disk.nvme0n1.out‚10346.45751953125 disk.drbd2265.in‚0) proc.memory.percent‚16.12982749938965 network.sent‚ 7.347632e+06 disk.drbd2265.out‚0 cpu‚25.284831674547767 disk.drbd1716.in‚0% disk./.usageGB‚299.38158416748047 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme0c0n1.in‚0.078125$ memory_percent‚95.54831981658936‚¿À2¡- _wandbruntime‚2716  _runtime‚2716º¦¬nÇ¢Á & lr-AdamW/pg1‚0.0001138937980131335 & lr-AdamW/pg2‚0.0001138937980131335 trainer/global_step‚3449 % _timestamp‚1.7884476012874043e+09 _runtime‚2716.759025146 _step‚278š£mDÌ£Æ % _timestamp‚1.7884476012874043e+09 _runtime‚2716.759025146 _step‚278 & lr-AdamW/pg1‚0.0001138937980131335 & lr-AdamW/pg2‚0.0001138937980131335 trainer/global_step‚3449–&º2¤- _wandbruntime‚2716  _runtime‚2716<ÏVºÝ'¥×O KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64815930 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177078784 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚201949271764480 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚220505338914816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 % _timestamp‚1.7884476012881482e+09 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚199075462137344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16392372 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚201949271764480 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 114560321 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49724352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29004804 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45397176 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚196745182843573 _step‚279 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49724354 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29004806 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚3449 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 114540282 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚201949946129408 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚199075462137344 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚196745854320685 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚2716.759963437 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64835967 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64835967 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚199075480140288 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16392806 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64815930 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2873809627136 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2863235327669 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2874465989120 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 153138688 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3017320181248 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚193881947515904 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚220505515993600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚199075480140288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚436 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2863889765421 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928a&oâ'¦ÜO eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚217488042673664 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2874465989120 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 153138688 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3017320181248 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚193881947515904 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚220505515993600 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚199075480140288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚436 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2863889765421 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚64815930 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177078784 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚201949271764480 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚220505338914816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 % _timestamp‚1.7884476012881482e+09 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚199075462137344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16392372 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚201949271764480 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 114560321 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚49724352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29004804 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45397176 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚196745182843573 _step‚279 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚49724354 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29004806 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚3449 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 114540282 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚201949946129408 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚199075462137344 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚196745854320685 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚2716.759963437 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚64835967 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚64835967 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚199075480140288 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16392806 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚64815930 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2873809627136 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2863235327669 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49738368 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 % _timestamp‚1.7884476019827633e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2864706200593 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚3449 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49738368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 114572752 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45410304 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _step‚280 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29012972 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16397330 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚193936851646464 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64834384 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚196800903409097 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611136 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚217549669471232 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64834384 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49738370 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114592817 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3018339106304 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚193936868685824 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114572752 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚220567844966400 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49738370 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45409871 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚202006444231168 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚199131815176192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2875285430272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114592817 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16396899 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚217549693411328 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚202007118609408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2875285430272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163611136 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚217549669471232 =7DeviceStatsMonitor.onuð£!_train_batch_end/segment.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚64834384 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚49738370 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 114592817 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3018339106304 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚193936868685824 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 114572752 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚220567844966400 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚49738370 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45409871 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚202006444231168 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚199131815176192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2875285430272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 114592817 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16396899 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚217549693411328 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚202007118609408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2875285430272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚49738368 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 % _timestamp‚1.7884476019827633e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2864706200593 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚3449 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚49738368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 114572752 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45410304 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _step‚280 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29012972 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16397330 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚193936851646464 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚64834384 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚196800903409097 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872˜8®92ª- _wandbruntime‚2717  _runtime‚2717£‘ùMV«Ð 0loss/FBt2/assignment_loss‚0.6135445147159772 /loss/FRt1/assignment_loss‚4.544570176689713 /loss/FRt4/assignment_loss‚3.192978614685582 /loss/FBt1/assignment_loss‚0.761283448764256 /loss/FRt1/detection_loss‚3.0369022775579384 1loss/SRqqt3/detection_loss‚1.3372847310442855 /loss/FBt1/detection_loss‚2.5394772035734996 1loss/SRqqt1/assignment_loss‚2.474551314967019 0loss/FBt4/assignment_loss‚0.4737366521731019 1loss/SRqqt1/detection_loss‚2.4202129534312657 _runtime‚2717.456001413 0loss/FRt3/assignment_loss‚2.1807588232415065 2loss/SRqqt3/assignment_loss‚1.4202306586169247 0loss/SRqqt2/detection_loss‚2.198446909063741 .loss/FBt4/detection_loss‚2.737842131406069 epoch‚2 /loss/FRt4/detection_loss‚2.3307180404663086 .loss/FBt3/detection_loss‚0.624418371161561 % _timestamp‚1.7884476019844642e+09 _step‚281 &loss/total_loss‚49.716215913129076 trainer/global_step‚3449 /loss/FRt2/assignment_loss‚4.922891869101414 1loss/SRqqt2/assignment_loss‚1.882498096086477 .loss/FRt2/detection_loss‚2.350346558315809 /loss/FRt3/detection_loss‚1.6084816229662724 0loss/SRqqt4/detection_loss‚1.462735195554072 2loss/SRqqt4/assignment_loss‚1.5290917777005704 1loss/FBt3/assignment_loss‚0.30598182200630597 /loss/FBt2/detection_loss‚2.7672321498394012nϲJ[¬Õ 1loss/SRqqt1/assignment_loss‚2.474551314967019 0loss/FBt4/assignment_loss‚0.4737366521731019 1loss/SRqqt1/detection_loss‚2.4202129534312657 _runtime‚2717.456001413 0loss/FRt3/assignment_loss‚2.1807588232415065 2loss/SRqqt3/assignment_loss‚1.4202306586169247 0loss/SRqqt2/detection_loss‚2.198446909063741 .loss/FBt4/detection_loss‚2.737842131406069 epoch‚2 /loss/FRt4/detection_loss‚2.3307180404663086 .loss/FBt3/detection_loss‚0.624418371161561 % _timestamp‚1.7884476019844642e+09 _step‚281 &loss/total_loss‚49.716215913129076 trainer/global_step‚3449 /loss/FRt2/assignment_loss‚4.922891869101414 1loss/SRqqt2/assignment_loss‚1.882498096086477 .loss/FRt2/detection_loss‚2.350346558315809 /loss/FRt3/detection_loss‚1.6084816229662724 0loss/SRqqt4/detection_loss‚1.462735195554072 2loss/SRqqt4/assignment_loss‚1.5290917777005704 1loss/FBt3/assignment_loss‚0.30598182200630597 /loss/FBt2/detection_loss‚2.7672321498394012 0loss/FBt2/assignment_loss‚0.6135445147159772 /loss/FRt1/assignment_loss‚4.544570176689713 /loss/FRt4/assignment_loss‚3.192978614685582 /loss/FBt1/assignment_loss‚0.761283448764256 /loss/FRt1/detection_loss‚3.0369022775579384 1loss/SRqqt3/detection_loss‚1.3372847310442855 /loss/FBt1/detection_loss‚2.5394772035734996™Ñüɹ“­‚(:ˆ þ–æÔÊѨý gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.307' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.2614 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÝaÓgK®‚(:À þ–æÔ«Þçÿ) disk./.usagePercent‚20.09247714078006 disk.sda3.in‚0 disk.sda3.out‚ 104.00390625) proc.memory.percent‚16.12982749938965 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.drbd2080.out‚0" proc.memory.rssMB‚ 2642.7109375% disk./.usageGB‚299.39242935180664 disk.drbd2266.out‚0 disk.sda.out‚ 104.00390625 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 14.396484375$ memory_percent‚95.55022716522217 cpu‚25.283627577110224 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda1.out‚0# disk.nvme0n1.out‚10400.86328125( proc.memory.availableMB‚ 729.05078125 network.recv‚986614 disk.dm-0.out‚10400.86328125 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 104.00390625 disk.drbd2265.out‚0 disk.drbd1716.in‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0 network.sent‚ 7.354234e+06 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda2.in‚0% disk.nvme0c0n1.out‚10400.86328125 disk.rbd0.in‚0 disk.sda1.in‚0 disk.nvme0c0n1.in‚0.078125Ö;cž¯‚(:“ —æÔתœý gpu.0.gpu‚41.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚149.015' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.802999999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0x±Po&°‚(:› —æÔò´¬ÿ disk.dm-0.in‚0.078125 disk.rbd0.in‚0 proc.cpu.threads‚17 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0" proc.memory.rssMB‚ 2642.7109375 network.sent‚ 7.462004e+06 disk.sda.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚104.5 disk.drbd2266.in‚0 disk.nvme0n1.out‚ 10441.6875 disk.sda3.in‚0 cpu‚25.28237327484325 network.recv‚991831$ disk./.usageGB‚299.4275436401367 disk.sda.out‚104.5 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0) proc.memory.percent‚16.12982749938965 disk.rbd1.out‚ 14.408203125 disk.dm-1.in‚0 disk.drbd2266.out‚0$ memory_percent‚95.54872512817383 disk.dm-0.out‚ 10441.6875 disk.dm-1.out‚104.5 disk.drbd2265.in‚0) disk./.usagePercent‚20.09483369013274 disk.rbd1.in‚0 disk.nvme0c0n1.in‚0.078125! disk.nvme0c0n1.out‚ 10441.6875& proc.memory.availableMB‚ 729.296875ІÛu2±- _wandbruntime‚2753  _runtime‚2753ü¹ Æ²À trainer/global_step‚3499 $ _timestamp‚1.788447638354799e+09 _runtime‚2753.826391457 _step‚282 & lr-AdamW/pg1‚0.0001133375282826524 & lr-AdamW/pg2‚0.0001133375282826524[ýß˳Š& lr-AdamW/pg2‚0.0001133375282826524 trainer/global_step‚3499 $ _timestamp‚1.788447638354799e+09 _runtime‚2753.826391457 _step‚282 & lr-AdamW/pg1‚0.0001133375282826524šLpª2´- _wandbruntime‚2753  _runtime‚2753§FnÝ'µ×O F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 116167134 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚196627171083264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65762019 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚223631045599744 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚201893132082688 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16620211 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _runtime‚2753.827342082 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3060489982976 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚201893114079744 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚423 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65741982 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2915451794944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65762019 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚196627154043904 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚201893114079744 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚204807909512704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16620634 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2914795432960 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50425152 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚204807909512704 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚201893132082688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2914795432960 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65741982 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _step‚283 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163624448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50425154 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚223631233164288 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚220570579556864 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚425 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46033415 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3060653607424 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414C¨Zú¤¶ÜO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚50425154 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29413206 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 trainer/global_step‚3499 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2915451794944 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚65741982 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _step‚283 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163624448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚50425154 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚223631233164288 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚220570579556864 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚425 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46033415 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3060653607424 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 116167134 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚196627171083264 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚65762019 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚223631045599744 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚201893132082688 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16620211 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _runtime‚2753.827342082 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3060489982976 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚201893114079744 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚423 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚65741982 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bêçŸð> ytes.small_pool.allocated‚ 2915451794944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚65762019 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚196627154043904 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚201893114079744 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚204807909512704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16620634 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2914795432960 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚50425152 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚204807909512704 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚201893132082688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2914795432960 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46046029 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116219669 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2904887102041 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16624657 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚201949485121536 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚223693559839232 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚220632230294528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 116219669 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116199604 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50439168 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3061353484800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚196682075213824 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚204865081979392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65760436 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚204865756357632 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65780499 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16625104 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚3499 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚201949467118592 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50439170 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2905541540001 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2915614860800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚223693747390464 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50439170 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚199587616753825 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65780499 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46046478 _runtime‚2754.517438232 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 116219669 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2904887102041 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16624657 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚201949485121536 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚223693559839232 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚220632230294528 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 116219669 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 116199604 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚50439168 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3061353484800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚196682075213824 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚204865081979392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚65760436 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚204865756357632 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚65780499 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16625104 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚3499 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚201949467118592 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚50439170 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2905541540001 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2915614860800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚223693747390464 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚50439170 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚199587616753825 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚65780499 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46046478 _runtime‚2754.517438232 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46046029 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0œŠ¯…á2º- _wandbruntime‚2754  _runtime‚2754\ë—UV»Ð 2loss/SRqqt3/assignment_loss‚1.3868728472213996 1loss/SRqqt2/assignment_loss‚2.176477804048253 2loss/SRqqt4/assignment_loss‚1.6296779893233742 /loss/FBt1/assignment_loss‚0.475102499127388 1loss/SRqqt4/detection_loss‚1.6614064071589791 0loss/FRt4/assignment_loss‚2.9825196663538613 2loss/SRqqt1/assignment_loss‚2.4181701613195017 /loss/FRt4/detection_loss‚2.4135819602580297 /loss/FBt3/detection_loss‚0.7120742477929672 trainer/global_step‚3499 -loss/FRt2/detection_loss‚2.66508956750234 0loss/FRt2/assignment_loss‚3.8055734175902147 0loss/FBt2/assignment_loss‚0.5900680638317388 1loss/SRqqt3/detection_loss‚1.4168292184670765 .loss/FBt1/detection_loss‚8.152343675494194 .loss/FBt4/detection_loss‚4.264454650878906 _step‚285 /loss/FRt1/assignment_loss‚4.842256232311851 0loss/FBt4/assignment_loss‚0.6971616864204406 /loss/FRt3/detection_loss‚1.9535546626961022 1loss/SRqqt1/detection_loss‚2.6166046912019905 0loss/SRqqt2/detection_loss‚1.821344113775662 $loss/total_loss‚57.1545947887912 epoch‚2 0loss/FRt3/assignment_loss‚2.4291251291308487 % _timestamp‚1.7884476390469406e+09 1loss/FBt3/assignment_loss‚0.27891200382343767 /loss/FRt1/detection_loss‚3.5686614889847603 .loss/FBt2/detection_loss‚2.196732604077884 _runtime‚2754.518446686Ëó[¼Õ 1loss/SRqqt3/detection_loss‚1.4168292184670765 .loss/FBt1/detection_loss‚8.152343675494194 .loss/FBt4/detection_loss‚4.264454650878906 _step‚285 /loss/FRt1/assignment_loss‚4.842256232311851 0loss/FBt4/assignment_loss‚0.6971616864204406 /loss/FRt3/detection_loss‚1.9535546626961022 1loss/SRqqt1/detection_loss‚2.6166046912019905 0loss/SRqqt2/detection_loss‚1.821344113775662 $loss/total_loss‚57.1545947887912 epoch‚2 0loss/FRt3/assignment_loss‚2.4291251291308487 % _timestamp‚1.7884476390469406e+09 1loss/FBt3/assignment_loss‚0.27891200382343767 /loss/FRt1/detection_loss‚3.5686614889847603 .loss/FBt2/detection_loss‚2.196732604077884 _runtime‚2754.518446686 2loss/SRqqt3/assignment_loss‚1.3868728472213996 1loss/SRqqt2/assignment_loss‚2.176477804048253 2loss/SRqqt4/assignment_loss‚1.6296779893233742 /loss/FBt1/assignment_loss‚0.475102499127388 1loss/SRqqt4/detection_loss‚1.6614064071589791 0loss/FRt4/assignment_loss‚2.9825196663538613 2loss/SRqqt1/assignment_loss‚2.4181701613195017 /loss/FRt4/detection_loss‚2.4135819602580297 /loss/FBt3/detection_loss‚0.7120742477929672 trainer/global_step‚3499 -loss/FRt2/detection_loss‚2.66508956750234 0loss/FRt2/assignment_loss‚3.8055734175902147 0loss/FBt2/assignment_loss‚0.5900680638317388R¼êž½‚(:“ œ—æÔ£‹ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.268' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.653600000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÂRKQ¾‚(:Æ œ—æÔÕ׃€% disk./.usageGB‚299.46519470214844 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda.in‚0 disk.sda1.in‚0 cpu‚25.28111094007542 disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 network.recv‚996942 disk.nvme0c0n1.in‚0.078125 disk.rbd1.out‚ 14.544921875 disk.sda3.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 105.16015625 disk.sda2.in‚0 disk.drbd1716.in‚0$ memory_percent‚95.54891586303711& proc.memory.availableMB‚ 729.265625 network.sent‚ 7.468492e+06 disk.dm-1.out‚ 105.16015625 proc.cpu.threads‚17) disk./.usagePercent‚20.09736048449569 disk.sda3.out‚ 105.16015625 disk.drbd2266.out‚0" proc.memory.rssMB‚ 2642.7109375 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚10469.11669921875 disk.dm-1.in‚0) proc.memory.percent‚16.12982749938965 disk.rbd0.out‚0# disk.dm-0.out‚10469.11669921875 disk.sda1.out‚0 disk.rbd1.in‚0& disk.nvme0n1.out‚10469.11669921875 disk.drbd1716.out‚0¿ª’¿‚(:’ «—æÔµ©“ý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚135.56' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.112000000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ˆ×©QÀ‚(:Æ «—æÔȵÿ disk.rbd0.in‚0 disk.rbd1.out‚ 14.669921875 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd2265.out‚0& proc.memory.availableMB‚ 729.265625 disk.drbd2080.in‚0 disk.drbd2080.out‚0$ memory_percent‚95.54891586303711" proc.memory.rssMB‚ 2642.7109375 cpu‚25.279889473505023$ disk./.usageGB‚299.4927215576172 disk.sda3.out‚ 105.953125 disk.sda1.out‚0 disk.dm-1.out‚ 105.953125 network.recv‚ 1.001889e+06 disk.sda3.in‚0 disk.sda.out‚ 105.953125& disk.nvme0n1.out‚10511.45654296875 disk.drbd1716.out‚0 proc.cpu.threads‚17 network.sent‚ 7.576087e+06 disk.dm-0.in‚0.078125# disk.dm-0.out‚10511.45654296875 disk.sda1.in‚0) disk./.usagePercent‚20.09920783486277 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0( disk.nvme0c0n1.out‚10511.45654296875 disk.sda2.out‚0 disk.drbd1716.in‚0) proc.memory.percent‚16.12982749938965 disk.drbd2266.in‚09cνžÁ‚(:“ º—æÔçãŽý gpu.0.gpu‚30.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚141.078' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.215600000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“µÒN‚(:à º—æÔ¢Âóþ disk.sda.out‚ 106.40625 disk.drbd1716.in‚0 cpu‚25.27867245380615 network.sent‚ 7.582766e+06 disk.sda3.out‚ 106.40625 disk.sda1.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125& proc.memory.availableMB‚ 729.015625" proc.memory.rssMB‚ 2642.7109375) proc.memory.percent‚16.12982749938965 disk.drbd2265.out‚0 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚10559.52490234375# disk.dm-0.out‚10559.52490234375 disk.sda2.out‚0$ disk./.usageGB‚299.5226821899414 disk.drbd2266.out‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.dm-1.in‚0* disk./.usagePercent‚20.101218518036692 disk.sda.in‚0$ memory_percent‚95.55044174194336 network.recv‚ 1.007171e+06 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd1.out‚ 14.732421875& disk.nvme0n1.out‚10559.52490234375 disk.drbd1716.out‚0 proc.cpu.threads‚17 disk.rbd0.out‚0 disk.sda1.in‚0 disk.dm-1.out‚ 106.40625÷™Ù`2Ã- _wandbruntime‚2790  _runtime‚2790ÖŠC@ÉÄà % _timestamp‚1.7884476749588466e+09 _runtime‚2790.430439582 _step‚286 ' lr-AdamW/pg1‚0.00011277083973539609 ' lr-AdamW/pg2‚0.00011277083973539609 trainer/global_step‚3549lвèÎÅÈ ' lr-AdamW/pg1‚0.00011277083973539609 ' lr-AdamW/pg2‚0.00011277083973539609 trainer/global_step‚3549 % _timestamp‚1.7884476749588466e+09 _runtime‚2790.430439582 _step‚286žkr¿2Æ- _wandbruntime‚2790  _runtime‚2790ü#—˜ÇÖO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚207667221625856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚51125954 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51125952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46669646 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758848 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚207667221625856 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚199372360571904 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66688071 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66668034 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚204710784025088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚199372377611264 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚204710766022144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚454 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29821604 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16848494 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818752 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66688071 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚207666547260928 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46670100 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2945560444237 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2944906006485 _runtime‚2790.431343929 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51125954 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚207666547260928 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚452 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16848042 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚226756671397888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 trainer/global_step‚3549˜´`á'ÈÛO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20037 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758848 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674364928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚207667221625856 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚199372360571904 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚66688071 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚66668034 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚204710784025088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674364928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚199372377611264 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚204710766022144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656361984 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚454 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29821604 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚16848494 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818752 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20039 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚66688071 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚207666547260928 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚46670100 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2945560444237 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2944906006485 _runtime‚2790.431343929 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51125954 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚207666547260928 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚452 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚16848042 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚226756671397888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24127 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 trainer/global_step‚3549 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚207667221625856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚51125954 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51125952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955585536 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24400 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚46669646 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0Ÿ^§Ÿ2É- _wandbruntime‚2791  _runtime‚2791/º/ç&ÊáM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚226819162923008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚66686488 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚204767119060992 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚204767137063936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 178291200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 _step‚288 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46682327 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚204767119060992 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚207724394105856 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66706551 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚202373658621233 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚202372987143913 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51139968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46682790 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51139970 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚3549 % _timestamp‚1.7884476756505592e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51139970 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66706551 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2957257041920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2945722441449 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117826456 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16852555 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚226819365154304 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚207724394105856 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66686488 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117846521 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _runtime‚2791.122669527 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚207723719727616 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3104419685376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2956600666624 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚223714767177728 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16853016 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3104597976576 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚204767137063936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2957257041920 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚199427281741824 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2956600666624 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 117826456 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29829774 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚199427264702464 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117846521 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚16853016 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3104597976576 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚204767137063936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2957257041920 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚199427281741824 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2956600666624 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 117826456 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29829774 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚199427264702464 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 117846521 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_tÞ¨ êMrain_batch_end/allocated_bytes.all.peak‚ 9955585536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 178291200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 _step‚288 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚46682327 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚204767119060992 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚207724394105856 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚66706551 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚202373658621233 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚202372987143913 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853517 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚51139968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815168 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚46682790 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚51139970 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚3549 % _timestamp‚1.7884476756505592e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51139970 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚66706551 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2957257041920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24127 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2945722441449 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 117826456 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚16852555 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚226819365154304 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20065 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20063 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚207724394105856 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚66686488 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 117846521 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24400 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _runtime‚2791.122669527 Â>4É2Ì- _wandbruntime‚2791  _runtime‚2791Õ\òVÍÐ % _timestamp‚1.7884476756523616e+09 /loss/FRt4/assignment_loss‚3.790536282459895 0loss/SRqqt1/assignment_loss‚2.65497541030248 2loss/SRqqt2/assignment_loss‚1.8582543475696673 2loss/SRqqt3/assignment_loss‚1.4165148381675994 0loss/FBt1/assignment_loss‚0.5377890503574235 2loss/FBt4/assignment_loss‚0.010067444112792145 _runtime‚2791.123862419 _step‚289 epoch‚2 1loss/FBt3/assignment_loss‚0.18374367489003587 /loss/FBt2/detection_loss‚2.1767812692202053 /loss/FBt4/detection_loss‚4.9968321323394775 /loss/FRt1/assignment_loss‚4.639385021649874 2loss/SRqqt4/assignment_loss‚1.5362809302142033 %loss/total_loss‚50.75758586547306 /loss/FRt3/detection_loss‚1.8319289049035625 1loss/SRqqt1/detection_loss‚2.5910485754410426 0loss/SRqqt2/detection_loss‚1.670717974121754 0loss/FBt2/assignment_loss‚0.5142812396470661 .loss/FRt2/detection_loss‚2.319065495285877 1loss/SRqqt4/detection_loss‚1.5166551573918416 .loss/FBt1/detection_loss‚2.320069116353989 .loss/FBt3/detection_loss‚1.006639092118104 .loss/FRt3/assignment_loss‚2.81013373220176 .loss/FRt1/detection_loss‚3.124633949536544 /loss/FRt4/detection_loss‚2.0055460878958304 trainer/global_step‚3549 /loss/FRt2/assignment_loss‚4.020555377006531 1loss/SRqqt3/detection_loss‚1.2251507622855051ë½q^[ÎÕ /loss/FRt2/assignment_loss‚4.020555377006531 1loss/SRqqt3/detection_loss‚1.2251507622855051 % _timestamp‚1.7884476756523616e+09 /loss/FRt4/assignment_loss‚3.790536282459895 0loss/SRqqt1/assignment_loss‚2.65497541030248 2loss/SRqqt2/assignment_loss‚1.8582543475696673 2loss/SRqqt3/assignment_loss‚1.4165148381675994 0loss/FBt1/assignment_loss‚0.5377890503574235 2loss/FBt4/assignment_loss‚0.010067444112792145 _runtime‚2791.123862419 _step‚289 epoch‚2 1loss/FBt3/assignment_loss‚0.18374367489003587 /loss/FBt2/detection_loss‚2.1767812692202053 /loss/FBt4/detection_loss‚4.9968321323394775 /loss/FRt1/assignment_loss‚4.639385021649874 2loss/SRqqt4/assignment_loss‚1.5362809302142033 %loss/total_loss‚50.75758586547306 /loss/FRt3/detection_loss‚1.8319289049035625 1loss/SRqqt1/detection_loss‚2.5910485754410426 0loss/SRqqt2/detection_loss‚1.670717974121754 0loss/FBt2/assignment_loss‚0.5142812396470661 .loss/FRt2/detection_loss‚2.319065495285877 1loss/SRqqt4/detection_loss‚1.5166551573918416 .loss/FBt1/detection_loss‚2.320069116353989 .loss/FBt3/detection_loss‚1.006639092118104 .loss/FRt3/assignment_loss‚2.81013373220176 .loss/FRt1/detection_loss‚3.124633949536544 /loss/FRt4/detection_loss‚2.0055460878958304 trainer/global_step‚3549¡ê—œÏ‚(:‘ É—æÔ¤´ ý gpu.0.gpu‚43.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚173.8' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚34.760000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0iËTVЂ(:Ë É—æÔ­ÿªÿ disk.sda.out‚ 106.8046875$ memory_percent‚95.54934501647949 disk.drbd2266.in‚0 disk.sda.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 proc.cpu.threads‚17 network.sent‚ 7.587929e+06) disk./.usagePercent‚20.10162096188052 disk.drbd1716.in‚0# disk.dm-0.out‚10597.64990234375 disk.sda1.in‚0 disk.sda2.in‚0' proc.memory.availableMB‚ 729.1953125 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0& disk.nvme0n1.out‚10597.64990234375% disk./.usageGB‚299.52867889404297 disk.sda2.out‚0 disk.dm-1.out‚ 106.8046875 network.recv‚ 1.010192e+06( disk.nvme0c0n1.out‚10597.64990234375 disk.sda3.in‚0 disk.sda3.out‚ 106.8046875 disk.drbd2266.out‚0 disk.rbd1.out‚ 14.732421875 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0" proc.memory.rssMB‚ 2642.7109375) proc.memory.percent‚16.12982749938965 disk.drbd2080.out‚0 cpu‚25.277542369713753Ýoh“Ñ‚(:ˆ Ø—æÔ–ŠÛý gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.724' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.9448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ùú¤XRÒ‚(:Ç Ø—æÔ’­ýÿ disk.drbd2266.in‚0 disk.sda.in‚0 disk.drbd1716.in‚0$ memory_percent‚95.54934501647949% disk.nvme0n1.out‚10647.5419921875 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.sda3.out‚ 107.3984375" proc.memory.rssMB‚ 2642.7109375 network.recv‚ 1.016851e+06$ disk./.usageGB‚299.5328941345215 disk.rbd0.out‚0 disk.drbd2265.in‚0' proc.memory.availableMB‚ 729.1953125 proc.cpu.threads‚17 disk.drbd2266.out‚0 disk.dm-1.in‚0 disk.drbd2080.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 14.7646484375 cpu‚25.2762448404386 disk.sda2.out‚0 disk.dm-1.out‚ 107.3984375 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚10647.5419921875 disk.dm-0.in‚0.078125 disk.sda.out‚ 107.3984375 disk.sda2.in‚0) proc.memory.percent‚16.12982749938965" disk.dm-0.out‚10647.5419921875 disk.sda1.in‚0 disk.sda1.out‚0 disk.rbd0.in‚0 network.sent‚ 7.696809e+06* disk./.usagePercent‚20.101903850205854™{˜í2Ó- _wandbruntime‚2827  _runtime‚2827î¼& ÈÔ ' lr-AdamW/pg1‚0.00011219385046840844 ' lr-AdamW/pg2‚0.00011219385046840844 trainer/global_step‚3599 $ _timestamp‚1.788447711636383e+09 _runtime‚2827.108082911 _step‚290(ÜÍÕÇ ' lr-AdamW/pg1‚0.00011219385046840844 ' lr-AdamW/pg2‚0.00011219385046840844 trainer/global_step‚3599 $ _timestamp‚1.788447711636383e+09 _runtime‚2827.108082911 _step‚290¢â322Ö- _wandbruntime‚2827  _runtime‚2827å&-!Ý'××O UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚210525859374080 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚210525185009152 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚67594086 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚207528417964544 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818752 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚202117584139264 % _timestamp‚1.7884477116371174e+09 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚226735629383168 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51826754 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚207528417964544 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3146581171712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚202117567099904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2996767044608 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚447 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2986395783645 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2985741345893 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2996767044608 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3146748990464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30230006 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚67594086 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚207528435967488 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 119440877 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚210525185009152 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 _runtime‚2827.108858345 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 119420838 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2997423406592 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚226735653323264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51826752 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47306072 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚205103308445797 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2997423406592 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚67614123 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30230004 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚205103979922909X:¿â'ØÜO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚210525185009152 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚67594086 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477112 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚207528417964544 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815168 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818752 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚202117584139264 % _timestamp‚1.7884477116371174e+09 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚226735629383168 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20039 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚51826754 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656361984 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚207528417964544 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3146581171712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 258008576 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699021 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚202117567099904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2996767044608 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚447 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2986395783645 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437752 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2985741345893 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2996767044608 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24127 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3146748990464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20037 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30230006 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚67594086 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚207528435967488 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 119440877 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚210525185009152 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955585536 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 _runtime‚2827.108858345 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 119420838 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2997423406592 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚226735653323264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚51826752 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚47306072 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚205103308445797 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24400 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2997423406592 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚67614123 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30230004 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚205103979922909 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853517 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚210525859374080£àN¾‰2Ù- _wandbruntime‚2827  _runtime‚2827•.ËSŸÚáM cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚226797304060928 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚229944728894464 B8DeviceStatsMonitor.on_train_batch_end/active.all.current=aYH%‚20065 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356928 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3147448773632 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51840770 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚210583031854080 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚207584771003392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2998242847744 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 119453308 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2997586472448 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47319180 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47318741 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚226797280120832 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2987212218817 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚210582357475840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3147608190464 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚207584789006336 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚67632603 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚67632603 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚202172488269824 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚207584789006336 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17080569 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚67612540 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2986557780857 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 119453308 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚67612540 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 trainer/global_step‚3599 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚202172471230464 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 _step‚292 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30238174 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚205159029011321 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2998242847744 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 119473373 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30238172 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416832 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2997586472448 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚207584789006336 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815168 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚17080569 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚67612540 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2986557780857 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 119453308 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20063 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚67612540 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 trainer/global_step‚3599 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚202172471230464 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955585536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 258008576 _step‚292 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30238174 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477320 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚205159029011321 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699021 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2998242847744 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 119473373 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30238172 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24400 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416832 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375296 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2997586472448 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375296 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚51840770 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚210583031854080 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚207584771003392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2998242847744 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955585536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 119453308 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2997586472448 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚47319180 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚47318741 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚226797280120832 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2987212218817 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚210582357475840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3147608190464 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚207584789006336 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚67632603 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚67632603 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚202172488269824 FDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 % _timestamp‚1.7884477806732829e+09 OIDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.current‚0 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914388 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 121221824 UHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚68489106 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 OADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚ 121221824 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚176 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 _step‚294 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚213948760022528 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚210910974463488 jVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.allocated‚230317784919552 BDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24127 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815168 SEDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚ 121237802 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15978 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24400 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15976 VIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚48197520 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚17420114 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚205384294009856 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15978 YLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚68489106 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.freed‚0 QIDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.allocated‚414 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚52748694 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 3038278388736 UHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚52748696 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚68473130 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚17419938 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 3026602812437 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚213949270855168 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚30777406 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚208410879782933 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 QDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚52748694 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9955585536 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853517 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 121237802 bNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚210910992466432 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚210910974463488 trainer/global_step‚3626 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 336208384ÁfS%I*ëÃT [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 158158848 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 [GDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated‚213949270855168 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚213948760022528 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚178 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 3188828548608 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚48197342 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _runtime‚2896.145223571 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚233506589528064 FDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 % _timestamp‚1.7884477806732829e+09 OIDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.current‚0 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914388 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 121221824 UHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚68489106 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 OADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚ 121221824 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚176 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 _step‚294 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚213948760022528 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚210910974463488 jVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.allocated‚230317784919552 BDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24127 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815168 SEDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚ 121237802 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15978 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24400 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15976 VIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚48197520 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚17420114 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚205384294009856 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15978 YLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚68489106 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.freed‚0 QIDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.allocated‚414 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚52748694 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 3038278388736 UHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚52748696 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚68473130 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚17419938 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 3026602812437 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚213949270855168 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚30777406 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚208410879782933 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 QDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚52748694 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9955585536 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853517 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 121237802 bNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚210910992466432 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚210910974463488 trainer/global_step‚3626 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 336208384 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 3027093726825 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510832640 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 134218752 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚30777404 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15976 UHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚68473130 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚208411387736681 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚205384276970496 LDDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.peak‚908 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 LCDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak‚2605 \JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚ 3037785559040 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 3037785559040 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 QGDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak‚24127 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚210910992466432 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 YLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚52748696 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚230317760979456 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955585536 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 3038278388736 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815168 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 3188962767360 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510832640 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492829696 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492829696¦äç¼™2ì- _wandbruntime‚2897  _runtime‚2897PǘÛM)íÇR cODeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.allocated‚205419587769344 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15978 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚213986466010624 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚210947822495232 XDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚213985955177984 aODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.allocated‚ 3038643515392 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15978 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚30783910 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853517 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 121255910 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 \NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚68496408 KADeviceStatsMonitor.on_validation_batch_end/active.small_pool.peak‚24127 TGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚48208335 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚30783908 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24400 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24400 SFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚52759500 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 3189327894016 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 RCDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.peak‚ 9643699021 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914388 _step‚295 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚210947822495232 YKDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.current‚ 158158848 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 134218752 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24400 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24400 SFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚52759500 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 3189327894016 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 RCDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.peak‚ 9643699021 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914388 _step‚295 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚210947822495232 YKDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.current‚ 158158848 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 134218752 E?De™¸üÁÎviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 B&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 7validation_average_jet_accuracy‚0.41914331912994385 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5024662613868713 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9642857313156128 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3928571343421936 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.5714285969734192 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.5208333134651184 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9944790601730347 (particle/accuracy‚0.8516709208488464 ("Purity/*FRt0SRqqt4FBt/event_purity‚0 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.1666666716337204 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 particle/accuracy_3_of_7‚1 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 7"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.39853435754776 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9504309892654419 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.03046446666121483 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 *jet/accuracy_2_of_6‚0.4166666865348816 jet/accuracy_5_of_7‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚1 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt*SRqqt4FBt/event_purity‚0 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3318629562854767 +jet/accuracy_1_of_1‚0.47097307443618774 ("Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0 & Purity/4FRt1SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.11861009895801544 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.11046908050775528 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5075700879096985 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.12758313119411469 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.5330459475517273 6 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.955920398235321 ;"Purity/1FRt*SRqqt0FBt/event_purity‚0.054305724799633026 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.7222222089767456 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9808721542358398 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.09183580428361893 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 particle/accuracy_2_of_5‚1 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.16126543283462524 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.14552192389965057 <"Purity/2FRt*SRqqt*FBt/event_purity‚0.0042252023704349995 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5544642210006714 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.4534785747528076 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.17238126695156097 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 7 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.4000000059604645 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8434551954269409 particle/accuracy_1_of_3‚1 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 _runtime‚2909.837826215 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.3945402503013611 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.48925021290779114 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4683907926082611 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.13418805599212646 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.4629629850387573 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.1255723088979721 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.625 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 (particle/f_score‚0.39948028326034546 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 jet/accuracy_3_of_7‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.4911302924156189 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.1875 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.029187319800257683 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0007936508045531809 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 ( Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 *particle/sensitivity‚0.316116064786911 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8333333134651184 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.23427453637123108 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 7"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.45333331823349 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9837315678596497 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.05186130851507187 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.3854166567325592 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 particle/accuracy_4_of_7‚1 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 6 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.4328521490097046 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.16391463577747345 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.41470590233802795 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.4759301245212555 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.014852607622742653 & Purity/3FRt4SRqqt*FBt/FRt_purity‚0 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9706158638000488 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9904297590255737 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.16936343908309937 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.15602219104766846 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.3854166865348816 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9799379110336304 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1182183176279068 + Purity/1FRt0SRqqt3FBt/FRt_purity‚0.5625 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.10383912175893784 particle/accuracy_6_of_6‚1 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 * Purity/3FRt3SRqqt*FBt/FRt_purity‚0.125 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3535838723182678 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.26230481266975403 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3192382752895355 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9351851940155029 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5218222737312317 *jet/accuracy_1_of_5‚0.9209164977073669 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.4826723039150238 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.4781671166419983 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.16647319495677948 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9166666269302368 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.5208333134651184 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.011503791436553001 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9808626174926758 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 ("Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6719598770141602 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 + Purity/2FRt2SRqqt1FBt/FRt_purity‚0.1875 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 +jet/accuracy_3_of_4‚0.04679689556360245 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.08433084934949875 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.17154951393604279 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0014430014416575432 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 +jet/accuracy_3_of_5‚0.09939195960760117 9"Purity/0FRt*SRqqt3FBt/event_purity‚0.1184210553765297 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.14393939077854156 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 jet/accuracy_2_of_7‚0 ) Purity/1FRt1SRqqt3FBt/FRt_purity‚0.25 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 6 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.980983316898346 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6297003626823425 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.03616352379322052 -jet/accuracy_4_of_4‚0.0030948990024626255 :"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.36666667461395264 ,"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.375 jet/accuracy_4_of_7‚0 (loss/val_total_loss‚50.3921035708834 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9981839656829834 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.3086419701576233 ) Purity/1FRt3SRqqt1FBt/FRt_purity‚0.25 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.13475176692008972 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5226914882659912 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11382970213890076 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 particle/accuracy_1_of_4‚1 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9853153824806213 6 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.986055850982666 particle/accuracy_1_of_7‚1 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.07015480101108551 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5116729736328125 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9856919646263123 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5277777910232544 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.2932291626930237 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 *jet/accuracy_2_of_5‚0.4525921642780304 +particle/specificity‚0.9507564306259155 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.4013403058052063 + Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9375 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 particle/accuracy_6_of_7‚1 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.17046907544136047 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.1746031641960144 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.11644144356250763 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 *"Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0.5 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.4431823492050171 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.10614035278558731 -"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3125 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 & Purity/4FRt*SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.34375 8"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.396093487739563 particle/accuracy_3_of_4‚1 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9576389789581299 7 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.2264367640018463 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.03392857313156128 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 particle/accuracy_2_of_3‚1 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 jet/accuracy_6_of_6‚0 particle/accuracy_5_of_6‚1 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.21322236955165863 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.12781040370464325 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 ("Purity/0FRt*SRqqt4FBt/event_purity‚0 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 jet/accuracy_7_of_7‚0 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5446876287460327 8"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.528069257736206 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 trainer/global_step‚3626 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9777939319610596 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.1666666716337204 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.11522633582353592 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9723484516143799 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.13696929812431335 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.3377193212509155 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3263888955116272 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9030733108520508 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 8 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.24166667461395264 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5166037678718567 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.20360930263996124 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 +"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.75 ) Purity/*FRt1SRqqt3FBt/FRt_purity‚0.25 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 particle/accuracy_1_of_6‚1 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.12154611945152283 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5009487271308899 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.4720321595668793 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5365842580795288 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.4505106210708618 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.30000001192092896 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.4995768368244171 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5614035129547119 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 _step‚296 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9853048324584961 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_1‚1 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 6 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.095238097012043 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 jet/accuracy_5_of_6‚0 & Purity/4FRt1SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.11790484189987183 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.15109871327877045 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.16930779814720154 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.4464285671710968 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 epoch‚2 6 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.328502893447876 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9976251721382141 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9929894208908081 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.8771929740905762 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.47699984908103943 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7272727489471436 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9732245802879333 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.1666666716337204 +"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.25 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.35791248083114624 ,jet/accuracy_4_of_5‚0.001792114693671465 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.39268583059310913 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.054597701877355576 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.29844963550567627 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 :"Purity/1FRt1SRqqtÌ^œ ó/*FBt/event_purity‚0.01490646880120039 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.11584702879190445 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_5‚1 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9908115267753601 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 jet/accuracy_5_of_5‚0 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.11777778714895248 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.46217474341392517 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9702380895614624 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1155330091714859 8"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.547641396522522 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.3950292766094208 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_4‚0.35965535044670105 7 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.4000000059604645 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.2348484992980957 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5063735842704773 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.44854792952537537 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.21819978952407837 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9877192974090576 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 :"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.47213852405548096 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_3‚1 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9715340733528137 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2708333134651184 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.26137566566467285 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.290554404258728 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_1_of_5‚1 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 particle/accuracy_4_of_6‚1 & Purity/3FRt4SRqqt0FBt/FRt_purity‚0 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.40127450227737427 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9954648017883301 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 & Purity/*FRt1SRqqt4FBt/FRt_purity‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.11539588123559952 ;"Purity/1FRt2SRqqt*FBt/event_purity‚0.000933706818614155 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.6969696879386902 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9976251721382141 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.12636053562164307 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_7‚1 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.2916666567325592 * Purity/3FRt3SRqqt0FBt/FRt_purity‚0.125 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 6 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.529411792755127 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9892290234565735 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5530785918235779 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5148103833198547 6 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.122909776866436 9"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.3597598075866699 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 +jet/accuracy_2_of_3‚0.22468161582946777 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.40666666626930237 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9879105687141418 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.014172336086630821 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.29372265934944153 ."Purity/0FRt0SRqqt3FBt/event_purity‚0.15625 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4178411066532135 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.2916666567325592 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.46963566541671753 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.07151389867067337 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.44384700059890747 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 *"Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0.5 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5115804672241211 +jet/accuracy_2_of_2‚0.09337131679058075 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 particle/accuracy_3_of_6‚1 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.0833333358168602 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.27464550733566284 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.4921301305294037 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 8 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.22962966561317444 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8109997510910034 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 ("Purity/0FRt0SRqqt4FBt/event_purity‚0 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.30000001192092896 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 % _timestamp‚1.7884477943641098e+09 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 +"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.25 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.4958218038082123 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.12257849425077438 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.30074405670166016 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 particle/accuracy_5_of_5‚1 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.4296700656414032 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.29305553436279297 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.13573437929153442 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.13333334028720856 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9990079402923584 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 particle/accuracy_4_of_4‚1 +jet/accuracy_3_of_6‚0.09615384787321091 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 jet/accuracy_6_of_7‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.2259661853313446 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.2349243313074112 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5406745672225952 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 & Purity/4FRt*SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.29256758093833923 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 jet/accuracy_1_of_6‚1 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.4901241064071655 ,"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.375 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9824270606040955 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.02612968347966671 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 *jet/accuracy_1_of_4‚0.9011215567588806 particle/accuracy_2_of_6‚1 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 jet/accuracy_1_of_7‚1 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.4787600636482239 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 ,jet/accuracy_3_of_3‚0.019271841272711754 jet/accuracy_4_of_6‚0 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9935897588729858 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 particle/accuracy_2_of_2‚1 particle/accuracy_3_of_5‚1 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.47874516248703003 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9206350445747375 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.18826621770858765 *jet/accuracy_1_of_2‚0.7251441478729248 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.4870590567588806 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 particle/accuracy_5_of_7‚1 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.11814603209495544 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.10909091681241989 particle/accuracy_2_of_4‚1 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1üÔ ÿOñÜ¥ ("Purity/2FRt1SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_5‚1 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9908115267753601 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 jet/accuracy_5_of_5‚0 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.11777778714895248 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.46217474341392517 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9702380895614624 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1155330091714859 8"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.547641396522522 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.3950292766094208 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_4‚0.35965535044670105 7 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.4000000059604645 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.2348484992980957 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5063735842704773 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.44854792952537537 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.21819978952407837 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9877192974090576 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 :"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.47213852405548096 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_3‚1 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9715340733528137 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2708333134651184 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.26137566566467285 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.290554404258728 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_1_of_5‚1 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 particle/accuracy_4_of_6‚1 & Purity/3FRt4SRqqt0FBt/FRt_purity‚0 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.40127450227737427 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9954648017883301 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 & Purity/*FRt1SRqqt4FBt/FRt_purity‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.11539588123559952 ;"Purity/1FRt2SRqqt*FBt/event_purity‚0.000933706818614155 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.6969696879386902 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9976251721382141 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.12636053562164307 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_7‚1 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.2916666567325592 * Purity/3FRt3SRqqt0FBt/FRt_purity‚0.125 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 6 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.529411792755127 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9892290234565735 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5530785918235779 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5148103833198547 6 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.122909776866436 9"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.3597598075866699 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 +jet/accuracy_2_of_3‚0.22468161582946777 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.40666666626930237 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9879105687141418 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.014172336086630821 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.29372265934944153 ."Purity/0FRt0SRqqt3FBt/event_purity‚0.15625 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4178411066532135 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.2916666567325592 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.46963566541671753 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.07151389867067337 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.44384700059890747 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 *"Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0.5 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5115804672241211 +jet/accuracy_2_of_2‚0.09337131679058075 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 particle/accuracy_3_of_6‚1 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.0833333358168602 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.27464550733566284 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.4921301305294037 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 8 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.22962966561317444 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8109997510910034 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 ("Purity/0FRt0SRqqt4FBt/event_purity‚0 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.30000001192092896 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 % _timestamp‚1.7884477943641098e+09 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 +"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.25 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.4958218038082123 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.12257849425077438 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.30074405670166016 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 particle/accuracy_5_of_5‚1 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.4296700656414032 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.29305553436279297 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.13573437929153442 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.13333334028720856 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9990079402923584 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 particle/accuracy_4_of_4‚1 +jet/accuracy_3_of_6‚0.09615384787321091 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 jet/accuracy_6_of_7‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.2259661853313446 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.2349243313074112 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5406745672225952 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 & Purity/4FRt*SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.29256758093833923 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 jet/accuracy_1_of_6‚1 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.4901241064071655 ,"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.375 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9824270606040955 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.02612968347966671 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 *jet/accuracy_1_of_4‚0.9011215567588806 particle/accuracy_2_of_6‚1 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 jet/accuracy_1_of_7‚1 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.4787600636482239 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 ,jet/accuracy_3_of_3‚0.019271841272711754 jet/accuracy_4_of_6‚0 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9935897588729858 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 particle/accuracy_2_of_2‚1 particle/accuracy_3_of_5‚1 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.47874516248703003 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9206350445747375 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.18826621770858765 *jet/accuracy_1_of_2‚0.7251441478729248 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.4870590567588806 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 particle/accuracy_5_of_7‚1 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.11814603209495544 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.10909091681241989 particle/accuracy_2_of_4‚1 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.2777777910232544 particle/accuracy_7_of_7‚1 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.4728473722934723 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.11580460518598557 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 9"Purity/*FRt0SRqqt0FBt/event_purity‚0.0992487445473671 :"Purity/*FRt1SRqqt2FBt/event_purity‚0.03169483318924904 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.25244763493537903 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.011283469386398792 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.4687613844871521 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5385416746139526 <"Purity/2FRt0SRqqt0FBt/event_purity‚0.0022675737272948027 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.20134396851062775 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.4995768368244171 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.4953337013721466 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 6 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.761904776096344 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.3747435510158539 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.41187363862991333 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.29629626870155334 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.11302682757377625 8"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.535493791103363 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9926449060440063 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.06239410489797592 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.26319825649261475 7 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.2907407879829407 7 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.9333333373069763 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1182183176279068 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5167282223701477 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 7validation_average_jet_accuracy‚0.41914331912994385 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5024662613868713 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9642857313156128 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3928571343421936 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.5714285969734192 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.5208333134651184 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9944790601730347 (particle/accuracy‚0.8516709208488464 ("Purity/*FRt0SRqqt4FBt/event_purity‚0 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.1666666716337204 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 particle/accuracy_3_of_7‚1 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 7"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.39853435754776 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9504309892654419 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.03046446666121483 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 *jet/accuracy_2_of_6‚0.4166666865348816 jet/accuracy_5_of_7‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚1 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt*SRqqt4FBt/event_purity‚0 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3318629562854767 +jet/accuracy_1_of_1‚0.47097307443618774 ("Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0 & Purity/4FRt1SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.11861009895801544 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.11046908050775528 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5075700879096985 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.12758313119411469 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.5330459475517273 6 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.955920398235321 ;"Purity/1FRt*SRqqt0FBt/event_purity‚0.054305724799633026 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.7222222089767456 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9808721542358398 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.09183580428361893 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 particle/accuracy_2_of_5‚1 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.16126543283462524 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.14552192389965057 <"Purity/2FRt*SRqqt*FBt/event_purity‚0.0042252023704349995 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5544642210006714 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.4534785747528076 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.17238126695156097 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 7 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.4000000059604645 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8434551954269409 particle/accuracy_1_of_3‚1 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 _runtime‚2909.837826215 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.3945402503013611 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.48925021290779114 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4683907926082611 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.13418805599212646 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.4629629850387573 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.1255723088979721 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.625 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 (particle/f_score‚0.39948028326034546 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 jet/accuracy_3_of_7‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.4911302924156189 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.1875 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 ("Purity/4FRt1SRqqžõqäBt*FBt/event_purity‚0 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.029187319800257683 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0007936508045531809 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 ( Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 *particle/sensitivity‚0.316116064786911 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8333333134651184 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.23427453637123108 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 7"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.45333331823349 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9837315678596497 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.05186130851507187 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.3854166567325592 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 particle/accuracy_4_of_7‚1 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 6 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.4328521490097046 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.16391463577747345 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.41470590233802795 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.4759301245212555 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.014852607622742653 & Purity/3FRt4SRqqt*FBt/FRt_purity‚0 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9706158638000488 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9904297590255737 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.16936343908309937 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.15602219104766846 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.3854166865348816 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9799379110336304 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1182183176279068 + Purity/1FRt0SRqqt3FBt/FRt_purity‚0.5625 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.10383912175893784 particle/accuracy_6_of_6‚1 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 * Purity/3FRt3SRqqt*FBt/FRt_purity‚0.125 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3535838723182678 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.26230481266975403 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3192382752895355 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9351851940155029 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5218222737312317 *jet/accuracy_1_of_5‚0.9209164977073669 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.4826723039150238 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.4781671166419983 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.16647319495677948 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9166666269302368 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.5208333134651184 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.011503791436553001 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9808626174926758 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 ("Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6719598770141602 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 + Purity/2FRt2SRqqt1FBt/FRt_purity‚0.1875 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 +jet/accuracy_3_of_4‚0.04679689556360245 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.08433084934949875 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.17154951393604279 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0014430014416575432 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 +jet/accuracy_3_of_5‚0.09939195960760117 9"Purity/0FRt*SRqqt3FBt/event_purity‚0.1184210553765297 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.14393939077854156 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 jet/accuracy_2_of_7‚0 ) Purity/1FRt1SRqqt3FBt/FRt_purity‚0.25 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 6 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.980983316898346 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6297003626823425 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.03616352379322052 -jet/accuracy_4_of_4‚0.0030948990024626255 :"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.36666667461395264 ,"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.375 jet/accuracy_4_of_7‚0 (loss/val_total_loss‚50.3921035708834 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9981839656829834 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.3086419701576233 ) Purity/1FRt3SRqqt1FBt/FRt_purity‚0.25 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.13475176692008972 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5226914882659912 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11382970213890076 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 particle/accuracy_1_of_4‚1 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9853153824806213 6 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.986055850982666 particle/accuracy_1_of_7‚1 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.07015480101108551 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5116729736328125 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9856919646263123 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5277777910232544 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.2932291626930237 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 *jet/accuracy_2_of_5‚0.4525921642780304 +particle/specificity‚0.9507564306259155 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.4013403058052063 + Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9375 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 particle/accuracy_6_of_7‚1 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.17046907544136047 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.1746031641960144 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.11644144356250763 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 *"Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0.5 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.4431823492050171 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.10614035278558731 -"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3125 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 & Purity/4FRt*SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.34375 8"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.396093487739563 particle/accuracy_3_of_4‚1 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9576389789581299 7 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.2264367640018463 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.03392857313156128 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 particle/accuracy_2_of_3‚1 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 jet/accuracy_6_of_6‚0 particle/accuracy_5_of_6‚1 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.21322236955165863 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.12781040370464325 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 ("Purity/0FRt*SRqqt4FBt/event_purity‚0 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 jet/accuracy_7_of_7‚0 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5446876287460327 8"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.528069257736206 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 trainer/global_step‚3626 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9777939319610596 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.1666666716337204 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.11522633582353592 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9723484516143799 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.13696929812431335 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.3377193212509155 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3263888955116272 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9030733108520508 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 8 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.24166667461395264 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5166037678718567 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.20360930263996124 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 +"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.75 ) Purity/*FRt1SRqqt3FBt/FRt_purity‚0.25 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 particle/accuracy_1_of_6‚1 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.12154611945152283 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5009487271308899 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.4720321595668793 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5365842580795288 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.4505106210708618 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.30000001192092896 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.4995768368244171 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5614035129547119 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 _step‚296 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9853048324584961 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_1‚1 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 6 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.095238097012043 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 jet/accuracy_5_of_6‚0 & Purity/4FRt1SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.11790484189987183 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.15109871327877045 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.16930779814720154 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.4464285671710968 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 epoch‚2 6 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.328502893447876 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9976251721382141 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9929894208908081 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.8771929740905762 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.47699984908103943 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7272727489471436 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9732245802879333 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.1666666716337204 +"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.25 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.35791248083114624 ,jet/accuracy_4_of_5‚0.001792114693671465 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.39268583059310913 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.054597701877355576 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.29844963550567627 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.01490646880120039 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.11584702879190445 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05¨\5åcò‚(:’ ²˜æÔ㵌þ gpu.0.gpu‚6.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚34.0 gpu.0.powerWatts‚122.865' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.572999999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0gŠî?ó‚(:´ ²˜æÔÛëìÿ$ disk.nvme0c0n1.out‚ 10906.4765625 disk.sda.out‚110.875 disk.sda1.out‚0 disk.dm-1.out‚110.875( proc.memory.availableMB‚ 729.56640625* disk./.usagePercent‚20.104809279711947 disk.drbd2080.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 network.recv‚ 1.043156e+06 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚110.875" disk.nvme0n1.out‚ 10906.4765625$ memory_percent‚95.54708003997803 cpu‚25.372408162794837 network.sent‚ 7.833791e+06 disk.sda.in‚0 disk.rbd1.out‚ 15.2578125 disk.drbd1716.out‚0 disk.drbd2266.out‚0 disk.dm-0.out‚ 10906.4765625 disk.sda2.out‚0 disk.drbd2265.in‚0# proc.memory.rssMB‚ 2642.48828125 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2265.out‚0 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.rbd0.out‚0) proc.memory.percent‚16.12846851348877% disk./.usageGB‚299.57618713378906ßK:_“ô‚(:ˆ Á˜æÔ©¥Îý gpu.0.gpu‚28.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚167.828' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.5656 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÓvÕ Mõ‚(: Á˜æÔ¨†Ïÿ disk.dm-0.out‚ 11418.8359375* disk./.usagePercent‚20.137089781719375 disk.dm-1.out‚ 111.16015625 disk.sda3.in‚0 proc.cpu.threads‚17 disk.sda1.in‚0 disk.drbd2266.out‚0 network.sent‚ 7.978277e+06 disk.sda.out‚ 111.16015625 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125$ memory_percent‚94.19882297515869 network.recv‚ 1.049163e+06% disk./.usageGB‚300.05718994140625 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.sda3.out‚ 111.16015625 disk.rbd0.out‚0# proc.memory.rssMB‚ 2642.48828125) proc.memory.percent‚16.12846851348877 disk.drbd2265.out‚0 disk.drbd2080.out‚0$ disk.nvme0c0n1.out‚ 11418.8359375 cpu‚25.36798259145115 disk.sda1.out‚0 disk.sda2.in‚0" disk.nvme0n1.out‚ 11418.8359375 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.drbd1716.in‚0( proc.memory.availableMB‚ 950.46484375 disk.sda.in‚0 disk.rbd1.out‚ 15.2578125 disk.drbd2265.in‚0¦Ó672ö- _wandbruntime‚2930  _runtime‚2930(RrüÉ÷à % _timestamp‚1.7884478151190174e+09 _runtime‚2930.590591016 _step‚297 ' lr-AdamW/pg1‚0.00011160668072538786 ' lr-AdamW/pg2‚0.00011160668072538786 trainer/global_step‚3649Á™‡ÎøÈ _runtime‚2930.590591016 _step‚297 ' lr-AdamW/pg1‚0.00011160668072538786 ' lr-AdamW/pg2‚0.00011160668072538786 trainer/global_step‚3649 % _timestamp‚1.7884478151190174e+09©C»Œ2ù- _wandbruntime‚2930  _runtime‚2930g™*¾Ý'ú×O cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3213079855616 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚68982099 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884478151196814e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚69002137 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48628589 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3050155623134 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3212903648768 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚215727134540800 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3060772819456 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176206848 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3049501185378 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31048158 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31048156 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48629038 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚232204337477120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3061429181952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3060772819456 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚232204361417216 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚298 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚215727134540800 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚207086280475648 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚3649 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53208330 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3061429181952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53208330 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440S‡üë;ûÜO SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚48629038 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚232204337477120 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3061429181952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3060772819456 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚232204361417216 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚298 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚215727134540800 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚207086280475648 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚3649 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53208330 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3061429181952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53208330 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3213079855616 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚68982099 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884478151196814e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚69002137 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚48628589 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3050155623134 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3212903648768 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚215727134540800 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3060772819456 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176206848 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3049501185378 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31048158 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31048156ªípA2ü- _wandbruntime‚2931  _runtime‚2931YˆÍNç&ýáM MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3050317620342 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 122242965 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 % _timestamp‚1.7884478158300676e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚215784981386240 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚212722714760192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122242965 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚232265988214784 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 122222899 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3062248623104 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚212722732763136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚235479935504384 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31056326 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚69020617 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3050972058306 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48641180 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53222346 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 176193536 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53222346 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚69000553 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 122222899 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17585313 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 _runtime‚2931.302217077 _step‚299 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200133632 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚207141184606208 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚215784307007488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚212722714760192 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3061592247296 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚69020617 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚457 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚459 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚215784981386240 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚212722714760192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 122242965 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚232265988214784 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 122222899 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3062248623104 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚212722732763136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚235479935504384 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31056326 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚69020617 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3050972058306 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚48641180 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚53222346 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 176193536 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚53222346 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚69000553 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 122222899 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17585313 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 _runtime‚2931.302217077 _step‚299 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200133632 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚207141184606208 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚215784307007488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚212722714760192 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3061592247296 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚69020617 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚457 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚459 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚215784981386240 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B«ku.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚138.361' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.672199999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚00TŒ9H…‚(:½ ߘæÔ§Ýçþ disk.sda1.out‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0' disk.nvme0c0n1.out‚11975.2451171875 disk.sda3.out‚ 112.21875 disk.rbd0.in‚0 disk.sda.out‚ 112.21875 disk.dm-1.in‚0 disk.sda3.in‚0 cpu‚25.364800344254387 network.sent‚ 8.09239e+06 disk.sda1.in‚0 disk.dm-1.out‚ 112.21875$ disk./.usageGB‚300.0978202819824 disk.drbd1716.out‚0 disk.drbd2265.in‚0% disk.nvme0n1.out‚11975.2451171875 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0$ memory_percent‚94.19903755187988 network.recv‚ 1.059414e+06 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2080.in‚0' proc.memory.availableMB‚ 950.4296875) proc.memory.percent‚16.12846851348877 disk.drbd2266.in‚0" disk.dm-0.out‚11975.2451171875 disk.rbd1.out‚15.53125 disk.nvme0c0n1.in‚0.078125# proc.memory.rssMB‚ 2642.48828125) disk./.usagePercent‚20.13981651796657 proc.cpu.threads‚17 Šu2†- _wandbruntime‚2967  _runtime‚2967­Ô½‹É‡Ã ' lr-AdamW/pg1‚0.00011100945287162875 ' lr-AdamW/pg2‚0.00011100945287162875 trainer/global_step‚3699 % _timestamp‚1.7884478519527838e+09 _runtime‚2967.424429613 _step‚301˜ÔÎÎˆÈ ' lr-AdamW/pg1‚0.00011100945287162875 ' lr-AdamW/pg2‚0.00011100945287162875 trainer/global_step‚3699 % _timestamp‚1.7884478519527838e+09 _runtime‚2967.424429613 _step‚301­ö Î2‰- _wandbruntime‚2967  _runtime‚2967"ûnÝ'Š×O UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3255721034240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚69908151 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚218585772289024 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚215484031666688 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚215484013663744 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚69908151 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3090336524786 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53909132 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3101758625280 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚218585772289024 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚238542785055744 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31456556 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚235286898300416 trainer/global_step‚3699 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚69928189 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚218586446654464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚238542595394560 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚218586446654464 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚209831487003648 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚69928189 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 123837321 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49260887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17804758 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _runtime‚2967.425323611 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53909130 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49261316 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31456558 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚212921823528434 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3102414987776 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17804331 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3090990962542 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 % _timestamp‚1.7884478519535148e+09 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 123817281 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53909130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚209831504043008 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3102414987776 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3255721034240 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚69908151 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚218585772289024 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚215484031666688 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚215484013663744 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚69908151 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3090336524786 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚53909132 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3101758625280 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚218585772289024 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚238542785055744 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31456556 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚235286898300416 trainer/global_step‚3699 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚69928189 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚218586446654464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚238542595394560 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚218586446654464 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚209831487003648 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚69928189 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 123837321 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49260887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚17804758 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _runtime‚2967.425323611 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚53909130 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49261316 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31456558 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚212921823528434 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3102414987776 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚17804331 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3090990962542 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 % _timestamp‚1.7884478519535148e+09 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 123817281 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚53909130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚209831504043008 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3102414987776 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17809275 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚235348525097984 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3256745977856 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _runtime‚2968.122545242 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚215540384705536 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49274001 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚218643619134464 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚215540366702592 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 123849751 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3103234428928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3103234428928 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚212978215571282 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚235348549038080 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚238605295015936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚69946669 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123869817 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚218642944755712 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚69946669 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3102578053120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3102578053120 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚209886408173568 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚218643619134464 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚238605109562368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚212977544093958 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123869817 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _step‚303 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚215540366702592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚431 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49273570 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123849751 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3103234428928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3103234428928 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚212978215571282 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚235348549038080 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚238605295015936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚69946669 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 123869817 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚218642944755712 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚69946669 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3102578053120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3102578053120 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚209886408173568 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚218643619134464 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚238605109562368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚212977544093958 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 123869817 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _step‚303 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚215540366702592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚431 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49273570 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 123849751 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚17809275 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚235348525097984 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3256745977856¯®#·2- _wandbruntime‚2968  _runtime‚2968°IÐCZÔ epoch‚3 2loss/SRqqt3/assignment_loss‚1.4031167649177474 .loss/FBt1/detection_loss‚3.295788808302446 trainer/global_step‚3699 /loss/FRt2/detection_loss‚2.3765927892105254 0loss/FRt2/assignment_loss‚3.5662447015444436 0loss/FBt1/assignment_loss‚0.3699364498720156 /loss/FBt3/detection_loss‚0.8174582744017244 2loss/SRqqt4/assignment_loss‚1.5132303969175727 0loss/FBt3/assignment_loss‚0.2438489846724666 /loss/FRt1/detection_loss‚3.1931313367990346 1loss/SRqqt2/detection_loss‚1.7909447252750397 _runtime‚2968.123635359 2loss/SRqqt1/assignment_loss‚2.5310702875808433 0loss/FBt4/assignment_loss‚0.9815356685448933 1loss/SRqqt3/detection_loss‚1.1899480010146526 1loss/SRqqt4/detection_loss‚1.7377669387669474 _step‚304 /loss/FRt1/assignment_loss‚5.130711207023034 .loss/FBt4/detection_loss‚4.726459339261055 2loss/SRqqt2/assignment_loss‚1.6813779498528068 /loss/FRt3/detection_loss‚2.0044338900882464 /loss/FRt4/detection_loss‚2.2673462300002574 %loss/total_loss‚52.54066772060243 0loss/SRqqt1/detection_loss‚2.940564274787903 .loss/FBt2/detection_loss‚3.295033817107861 % _timestamp‚1.7884478526521585e+09 0loss/FRt3/assignment_loss‚1.8423139716570194 0loss/FRt4/assignment_loss‚3.4140355706214907 0loss/FBt2/assignment_loss‚0.2277773423824057®f_‘Ù /loss/FRt2/detection_loss‚2.3765927892105254 0loss/FRt2/assignment_loss‚3.5662447015444436 0loss/FBt1/assignment_loss‚0.3699364498720156 /loss/FBt3/detection_loss‚0.8174582744017244 2loss/SRqqt4/assignment_loss‚1.5132303969175727 0loss/FBt3/assignment_loss‚0.2438489846724666 /loss/FRt1/detection_loss‚3.1931313367990346 1loss/SRqqt2/detection_loss‚1.7909447252750397 _runtime‚2968.123635359 2loss/SRqqt1/assignment_loss‚2.5310702875808433 0loss/FBt4/assignment_loss‚0.9815356685448933 1loss/SRqqt3/detection_loss‚1.1899480010146526 1loss/SRqqt4/detection_loss‚1.7377669387669474 _step‚304 /loss/FRt1/assignment_loss‚5.130711207023034 .loss/FBt4/detection_loss‚4.726459339261055 2loss/SRqqt2/assignment_loss‚1.6813779498528068 /loss/FRt3/detection_loss‚2.0044338900882464 /loss/FRt4/detection_loss‚2.2673462300002574 %loss/total_loss‚52.54066772060243 0loss/SRqqt1/detection_loss‚2.940564274787903 .loss/FBt2/detection_loss‚3.295033817107861 % _timestamp‚1.7884478526521585e+09 0loss/FRt3/assignment_loss‚1.8423139716570194 0loss/FRt4/assignment_loss‚3.4140355706214907 0loss/FBt2/assignment_loss‚0.2277773423824057 epoch‚3 2loss/SRqqt3/assignment_loss‚1.4031167649177474 .loss/FBt1/detection_loss‚3.295788808302446 trainer/global_step‚3699°¨C¡Ì“’‚(:ˆ î˜æÔ¤ù„ý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚165.746' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.1492 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ǬaP“‚(:Å î˜æÔ •€ network.sent‚ 8.098997e+06 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda3.out‚ 112.828125 cpu‚25.36328630956433% disk./.usageGB‚300.12265396118164 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 112.828125% disk.nvme0n1.out‚12023.8173828125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2080.out‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17 network.recv‚ 1.064672e+06 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0# proc.memory.rssMB‚ 2566.47265625 disk.sda.in‚0 disk.sda1.out‚0' disk.nvme0c0n1.out‚12023.8173828125) proc.memory.availableMB‚ 1007.81640625* proc.memory.percent‚15.664505958557129 disk.drbd2266.out‚0" disk.dm-0.out‚12023.8173828125 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.out‚15.59375 disk.dm-1.out‚ 112.828125 disk.sda3.in‚0* disk./.usagePercent‚20.141483127014475 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0$ memory_percent‚93.84877681732178¡¢^¬“”‚(:ˆ ý˜æÔ§üìü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.868' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9736 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0^˜újS•‚(:È ý˜æÔÒ®„ÿ$ memory_percent‚96.65687084197998) proc.memory.percent‚15.67540168762207 disk.drbd2266.in‚0* disk./.usagePercent‚20.141672572589812 disk.sda2.out‚0 disk.drbd2265.in‚0 disk.rbd0.out‚0$ disk./.usageGB‚300.1254768371582 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚12062.9541015625" disk.dm-0.out‚12062.9541015625 disk.sda.in‚0 disk.drbd1716.in‚0 disk.drbd2265.out‚0 cpu‚25.361663237130653 disk.sda.out‚ 113.46484375 disk.sda2.in‚0 disk.drbd1716.out‚0 proc.cpu.threads‚17 disk.sda3.out‚ 113.46484375 disk.rbd0.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚12062.9541015625 disk.dm-1.out‚ 113.46484375 network.sent‚ 8.206591e+06 disk.rbd1.out‚15.59375" proc.memory.rssMB‚ 2568.2578125 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 network.recv‚ 1.070147e+06 disk.dm-0.in‚0.078125 disk.sda1.out‚0( proc.memory.availableMB‚ 547.73828125 disk.drbd2266.out‚0 disk.sda1.in‚0uXž–‚(:“ Œ™æÔ”¥«ý gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚186.837' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.367399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0§P­W—‚(:Ì Œ™æÔ²¦Íÿ" proc.memory.rssMB‚ 2568.2578125* disk./.usagePercent‚20.141881218730216 disk.rbd0.in‚0 disk.sda.out‚ 114.21484375 disk.dm-1.out‚ 114.21484375 disk.drbd2266.in‚0) proc.memory.percent‚15.67540168762207 disk.rbd0.out‚0 disk.sda.in‚0 cpu‚25.360110056448722 disk.drbd2265.out‚0$ disk./.usageGB‚300.1285858154297 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd2080.out‚0 network.recv‚ 1.07531e+06 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17 network.sent‚ 8.213025e+06 disk.dm-0.in‚0.078125# disk.dm-0.out‚12114.11767578125 disk.rbd1.in‚0 disk.rbd1.out‚ 15.7421875 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 114.21484375 disk.sda2.in‚0 disk.drbd1716.in‚0( disk.nvme0c0n1.out‚12114.11767578125 disk.sda3.in‚0$ memory_percent‚96.65863513946533& disk.nvme0n1.out‚12114.11767578125 disk.dm-1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0( proc.memory.availableMB‚ 547.44921875VRÍ2˜- _wandbruntime‚3004  _runtime‚3004õ^^Ç™Á _runtime‚3004.293049977 _step‚305 & lr-AdamW/pg1‚0.0001104022913685207 & lr-AdamW/pg2‚0.0001104022913685207 trainer/global_step‚3749 % _timestamp‚1.7884478888214176e+09ÀÌšÆ trainer/global_step‚3749 % _timestamp‚1.7884478888214176e+09 _runtime‚3004.293049977 _step‚305 & lr-AdamW/pg1‚0.0001104022913685207 & lr-AdamW/pg2‚0.0001104022913685207±"nˇ2›- _wandbruntime‚3004  _runtime‚3004çüTzÜ'œÖO TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884478888221083e+09 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚212576710571008 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚218301665606144 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3143400793600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3142744431104 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚221444410037248 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3298491593728 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31864958 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 125444133 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚221445084402688 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚241667902837248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚238369435183616 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54609932 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3131826301950 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚54609930 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54609932 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚221445084402688 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31864956 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚70834203 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 125464173 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3131171864194 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚218301683609088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚215707865395842 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3142744431104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49895613 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3298663606272 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚218301683609088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚70834203 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚221444410037248 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚306 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 trainer/global_step‚3749 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚70854241 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54609930 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚70854241 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚241668098789888 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚215708536872958 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _runtime‚ 3004.29387334 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚218301665606144 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49895175 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18030655 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18030219 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 8618535217Íþ;UÛO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚218301683609088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚215707865395842 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3142744431104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚49895613 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3298663606272 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚218301683609088 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚70834203 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚221444410037248 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚306 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 trainer/global_step‚3749 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚70854241 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚54609930 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚70854241 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pd‡ÑŒool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚241668098789888 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚215708536872958 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _runtime‚ 3004.29387334 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚218301665606144 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚49895175 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18030655 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18030219 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884478888221083e+09 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚212576710571008 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚218301665606144 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3143400793600 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3142744431104 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚221444410037248 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3298491593728 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31864958 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 125444133 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚221445084402688 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚241667902837248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚238369435183616 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚54609932 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3131826301950 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚54609930 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚54609932 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚221445084402688 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31864956 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚70834203 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 125464173 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3131171864194²¾÷`X2ž- _wandbruntime‚3004  _runtime‚3004­§?"ç&ŸáM REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31873124 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3143563858944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3144220234752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3144220234752 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚215763585961366 % _timestamp‚1.7884478895032651e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3143563858944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚70872721 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚238431085921280 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31873126 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚441 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚70852657 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚3749 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚221502256882688 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚212631597662208 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 125496669 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚443 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49907766 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3299346749952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3131988299158 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚241730598379008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚212631614701568 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚218358036647936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚54623948 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49908209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 125496669 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚241730408731136 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚70872721 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54623946 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚218358018644992 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚221502256882688 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚54623948 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚70852657 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401v‘wâì& æM KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚238431061981184 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 125476603 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚3749 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚221502256882688 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚212631597662208 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 125496669 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚443 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚49907766 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3299346749952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3131988299158 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚241730598379008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚212631614701568 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚218358036647936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚54623948 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚49908209 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 125496669 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚241730408731136 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚70872721 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚54623946 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚218358018644992 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚221502256882688 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚54623948 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚70852657 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31873124 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3143563858944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3144220234752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3144220234752 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚215763585961366 % _timestamp‚1.7884478895032651e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3143563858944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚70872721 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚238431085921280 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31873126 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚441 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚70852657 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 $ _timestamp‚1.788447925877963e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.Áu½º‰#on_train_batch_start/allocated_bytes.small_pool.freed‚ 3183730236928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚450 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18256604 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 127091025 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200146944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚310 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚221119335551488 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚241451972066816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50529960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚224303047785472 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 trainer/global_step‚3799 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚241451948126720 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚221119335551488 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚224303722150912 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3183730236928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 127070985 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18257052 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3341571695616 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚71760255 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚221119317548544 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚218494578740366 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚218493907263250 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55310732 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3184386599424 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ˜‘òá'­ÛO ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176206848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3172007203602 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚50530410 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3172661641358 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚55310730 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚71780293 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚244793543762432 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚3041.349758432 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3184386599424 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚215321917099008 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚215321900059648 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚71780293 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚55310730 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3341395488768 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚224303047785472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32273358 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚221119335551488 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚241451972066816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚50529960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚224303047785472 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 trainer/global_step‚3799 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚241451948126720 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚221119335551488 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚224303722150912 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3183730236928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 127070985 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18257052 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3341571695616 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚71760255 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚221119317548544 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚218494578740366 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚218493907263250 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚55310732 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3184386599424 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚71760255 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 $ _timestamp‚1.788447925877963e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3183730236928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚450 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18256604 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 127091025 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200146944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚310 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3173478076530 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3184549664768 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚241513598864384 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚71778709 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3185206040576 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50542646 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚224360220252160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 _runtime‚3042.046218088 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3172823638566 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 127103455 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55324746 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18261555 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚241513622804480 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚244856036732416 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3342239831552 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚221175670587392 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚218550299306098 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3184549664768 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32281524 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18261122 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 127123521 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚221175688590336 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 % _timestamp‚1.7884479265741248e+09 _step‚311 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55324746 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚224360894630912 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚221175670587392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚244855838695936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚221175688590336 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32281526 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚215376804190208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55324748 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3342413927936 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50543081 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127123521 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0IËZ °æM EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 127123521 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚55324748 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚71798773 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3173478076530 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3184549664768 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚241513598864384 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚71778709 I@DeviceStatsMonitor.on_train_ba`™*µ’tch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3185206040576 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚50542646 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚224360220252160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 _runtime‚3042.046218088 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3172823638566 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 127103455 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚55324746 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18261555 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚241513622804480 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚244856036732416 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3342239831552 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚221175670587392 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚218550299306098 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3184549664768 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚32281524 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18261122 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 127123521 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚221175688590336 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 % _timestamp‚1.7884479265741248e+09 _step‚311 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚55324746 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚224360894630912 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚221175670587392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚244855838695936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚221175688590336 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32281526 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚215376804190208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚55324748 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3342413927936 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚50543081 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0·¿62±- _wandbruntime‚3042  _runtime‚3042T³DñX²Ò 1loss/FBt3/assignment_loss‚0.23528832670897507 1loss/FBt4/assignment_loss‚0.00875241444009589 /loss/FBt3/detection_loss‚0.7697520292937182 .loss/FBt4/detection_loss‚5.840328812599182 0loss/FRt3/assignment_loss‚1.8779214309020476 /loss/FRt1/detection_loss‚3.3839551080828127 0loss/SRqqt4/detection_loss‚1.682455364200804 /loss/FRt2/assignment_loss‚3.579724109691122 .loss/FRt4/detection_loss‚2.589847142828835 trainer/global_step‚3799 _runtime‚3042.047195452 % _timestamp‚1.7884479265757017e+09 0loss/FRt1/assignment_loss‚5.4496736733809765 1loss/SRqqt1/assignment_loss‚2.811235091902993 epoch‚3 0loss/SRqqt2/detection_loss‚1.605506767553312 .loss/FBt2/detection_loss‚2.193164245949851 %loss/total_loss‚52.69892247344022 2loss/SRqqt3/assignment_loss‚1.0278938703704625 .loss/FBt1/detection_loss‚3.365149575471878 0loss/FBt2/assignment_loss‚0.4964170679446498 /loss/FRt2/detection_loss‚2.1304406679194905 1loss/SRqqt1/detection_loss‚2.9231364239345896 0loss/FRt4/assignment_loss‚3.5941051416926912 2loss/SRqqt2/assignment_loss‚1.9480566169238753 2loss/SRqqt4/assignment_loss‚1.8121233916393034 0loss/FBt1/assignment_loss‚0.3185105069715064 /loss/FRt3/detection_loss‚1.9090086033737117 0loss/SRqqt3/detection_loss‚1.146476089663338 _step‚312½d[°]³× 0loss/FRt3/assignment_loss‚1.8779214309020476 /loss/FRt1/detection_loss‚3.3839551080828127 0loss/SRqqt4/detection_loss‚1.682455364200804 /loss/FRt2/assignment_loss‚3.579724109691122 .loss/FRt4/detection_loss‚2.589847142828835 trainer/global_step‚3799 _runtime‚3042.047195452 % _timestamp‚1.7884479265757017e+09 0loss/FRt1/assignment_loss‚5.4496736733809765 1loss/SRqqt1/assignment_loss‚2.811235091902993 epoch‚3 0loss/SRqqt2/detection_loss‚1.605506767553312 .loss/FBt2/detection_loss‚2.193164245949851 %loss/total_loss‚52.69892247344022 2loss/SRqqt3/assignment_loss‚1.0278938703704625 .loss/FBt1/detection_loss‚3.365149575471878 0loss/FBt2/assignment_loss‚0.4964170679446498 /loss/FRt2/detection_loss‚2.1304406679194905 1loss/SRqqt1/detection_loss‚2.9231364239345896 0loss/FRt4/assignment_loss‚3.5941051416926912 2loss/SRqqt2/assignment_loss‚1.9480566169238753 2loss/SRqqt4/assignment_loss‚1.8121233916393034 0loss/FBt1/assignment_loss‚0.3185105069715064 /loss/FRt3/detection_loss‚1.9090086033737117 0loss/SRqqt3/detection_loss‚1.146476089663338 _step‚312 1loss/FBt3/assignment_loss‚0.23528832670897507 1loss/FBt4/assignment_loss‚0.00875241444009589 /loss/FBt3/detection_loss‚0.7697520292937182 .loss/FBt4/detection_loss‚5.840328812599182¸¤Æé “´‚(:ˆ ¹™æÔËëþü gpu.0.gpu‚45.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚147.931' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚29.5862 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Þx"êWµ‚(:Ì ¹™æÔó¨’ÿ% disk./.usageGB‚300.14209365844727 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.rbd0.out‚0& disk.nvme0n1.out‚12244.79248046875 disk.drbd1716.out‚0 disk.sda3.in‚0 disk.sda3.out‚ 116.05859375 disk.sda2.in‚0 network.sent‚ 8.332642e+06 disk.rbd0.in‚0# disk.dm-0.out‚12244.79248046875$ memory_percent‚96.66228294372559 disk.dm-1.out‚ 116.05859375 disk.sda1.out‚0 disk.drbd2080.out‚0 network.recv‚ 1.089864e+06 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚0( disk.nvme0c0n1.out‚12244.79248046875 disk.sda.out‚ 116.05859375' proc.memory.availableMB‚ 546.8515625 cpu‚25.35549720077337 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.rbd1.out‚ 16.1015625* proc.memory.percent‚15.675878524780273 proc.cpu.threads‚17) disk./.usagePercent‚20.14278774140896 disk.drbd2266.in‚0 disk.sda1.in‚0" proc.memory.rssMB‚ 2568.3359375ªÚ÷“¶‚(:ˆ È™æÔú¬Çþ gpu.0.gpu‚33.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.082' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2164 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0µz¢ŠY·‚(:Î È™æÔÔ‘®ÿ( proc.memory.availableMB‚ 546.59765625 disk.sda1.out‚0 disk.drbd2266.in‚0# disk.dm-0.out‚12288.27783203125 disk.sda2.in‚0 proc.cpu.threads‚17) disk./.usagePercent‚20.14606028571915& disk.nvme0n1.out‚12288.27783203125 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda1.in‚0* proc.memory.percent‚15.675878524780273 cpu‚25.354010102911822 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.rbd1.out‚ 16.1337890625$ memory_percent‚96.66383266448975 network.recv‚ 1.096512e+06% disk./.usageGB‚300.19085693359375 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 network.sent‚ 8.441478e+06 disk.drbd1716.in‚0 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚12288.27783203125 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.dm-1.out‚ 116.6171875 disk.sda3.out‚ 116.6171875" proc.memory.rssMB‚ 2568.3359375 disk.dm-1.in‚0 disk.sda.in‚0 disk.sda.out‚ 116.6171875 disk.rbd1.in‚0dÛö)“¸‚(:ˆ ×™æÔºì¢ý gpu.0.gpu‚37.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.766' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5532 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€›M¹‚(: יæÔ˧ÿ# disk.nvme0n1.out‚12332.72265625 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2265.in‚0 network.recv‚ 1.099649e+06 disk.sda.in‚0 disk.rbd1.out‚ 16.2587890625 disk.drbd2080.out‚0% disk.nvme0c0n1.out‚12332.72265625 disk.drbd2266.out‚0 disk.rbd0.out‚0# memory_percent‚96.6623306274414 disk.sda2.out‚0# proc.memory.rssMB‚ 2570.11328125 cpu‚25.352461473573413$ disk./.usageGB‚300.2193412780762 disk.dm-0.in‚0.078125 disk.dm-0.out‚12332.72265625 disk.sda.out‚ 117.1328125 disk.dm-1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.sda3.out‚ 117.1328125 disk.drbd2266.in‚0* proc.memory.percent‚15.686726570129395 network.sent‚ 8.446633e+06 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 117.1328125 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 proc.cpu.threads‚17* disk./.usagePercent‚20.147971893977328 disk.drbd2265.out‚0 disk.rbd0.in‚0% proc.memory.availableMB‚ 546.84375Õž»2º- _wandbruntime‚3078  _runtime‚3078DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚223936969490944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32681756 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚434 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚72706345 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3225372405248 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚72706345 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56011532 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56011530 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚227161685533696 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚72686307 _step‚314 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3225372405248 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚227161685533696 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚244534508950016 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51165921 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56011532 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚244534485009920 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚223936987493888 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚223936987493888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32681758 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3384347576320 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚218067106587648 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚227162359899136 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚247918856526336 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3213496980766 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3212842543010 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18483729 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚247918671059456 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51165485 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18484163 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3224716042752 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 128697837 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038øЄ‰¿ÜO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚72686307 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚223936969490944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚32681756 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚434 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚72706345 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3225372405248 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚72706345 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56011532 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56011530 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚227161685533696 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚72686307 _step‚314 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3225372405248 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚227161685533696 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚244534508950016 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51165921 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56011532 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚244534485009920 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚223936987493888 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚223936987493888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚32681758 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3384347576320 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚218067106587648 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚227162359899136 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚247918856526336 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3213496980766 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3212842543010 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18483729 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚247918671059456 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51165485 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18484163 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3224716042752 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 128697837º]²@2À- _wandbruntime‚3078  _runtime‚3078{“¬:æ&ÁàM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚223993340532736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3214313415938 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51178144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56025548 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚72724825 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚218122027757568 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56025546 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚223993322529792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚218122010718208 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚244596135747584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56025546 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚3849 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚72704761 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32689926 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 128730307 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18488666 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚247981175932416 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚227218858000384 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3225535470592 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _runtime‚ 3078.97984341 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚221336341173506 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3225535470592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3213658977974 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚223993322529792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128750373 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56025548 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚227219532379136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18488220 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚56025548 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚72724825 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚218122027757568 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56025546 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚223993322529792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚218122010718208 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚244596135747584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚56025546 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚3849 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚72704761 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚32689926 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 128730307 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚18488666 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚247981175932416 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚227218858000384 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3225535470592 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _runtime‚ 3078.97984341 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚221336341173506 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3225535470592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3213658977974 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚223993322529792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 128750373 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚56025548 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚227219532379136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18488220 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51178144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424»)ÅÙM2Ã- _wandbruntime‚3078  _runtime‚3078Pbo^]Ä× /loss/FRt2/assignment_loss‚2.999085804820061 1loss/SRqqt4/assignment_loss‚2.245797117373773 1loss/FBt3/assignment_loss‚0.18374991031193552 /loss/FRt2/detection_loss‚2.1352452754974367 /loss/FBt1/detection_loss‚3.3932648572054775 /loss/FBt4/detection_loss‚3.8088075280189515 epoch‚3 % _timestamp‚1.7884479635093873e+09 /loss/FRt1/detection_loss‚3.0587422231148027 .loss/FRt3/detection_loss‚2.008841513718168 1loss/SRqqt2/detection_loss‚2.0578879630565643 0loss/FRt3/assignment_loss‚2.4913122773170473 2loss/SRqqt2/assignment_loss‚1.3449529764056205 2loss/SRqqt3/assignment_loss‚1.0948221434609613 1loss/SRqqt3/detection_loss‚1.3702115714550018 1loss/SRqqt4/detection_loss‚1.7274845157350813 /loss/FBt3/detection_loss‚0.8138644903976943 1loss/FBt1/assignment_loss‚0.33720319704380847 0loss/FBt2/assignment_loss‚0.3979648575163372 1loss/FBt4/assignment_loss‚0.10788284467998892 trainer/global_step‚3849 /loss/FRt4/detection_loss‚2.5046517627036318 1loss/SRqqt1/assignment_loss‚2.515125643323969 1loss/SRqqt1/detection_loss‚2.8044646437521332 %loss/total_loss‚49.68892705072647 _runtime‚3078.981192667 /loss/FRt1/assignment_loss‚5.221640167565181 0loss/FRt4/assignment_loss‚2.8317785719607738 /loss/FBt2/detection_loss‚2.2341451942920685 _step‚316/ÅŒbÅÜ 1loss/SRqqt4/detection_loss‚1.7274845157350813 /loss/FBt3/detection_loss‚0.8138644903976943 1loss/FBt1/assignment_loss‚0.33720319704380847 0loss/FBt2/assignment_loss‚0.3979648575163372 1loss/FBt4/assignment_loss‚0.10788284467998892 trainer/global_step‚3849 /loss/FRt4/detection_loss‚2.5046517627036318 1loss/SRqqt1/assignment_loss‚2.515125643323969 1loss/SRqqt1/detection_loss‚2.8044646437521332 %loss/total_loss‚49.68892705072647 _runtime‚3078.981192667 /loss/FRt1/assignment_loss‚5.221640167565181 0loss/FRt4/assignment_loss‚2.8317785719607738 /loss/FBt2/detection_loss‚2.2341451942920685 _step‚316 /loss/FRt2/assignment_loss‚2.999085804820061 1loss/SRqqt4/assignment_loss‚2.245797117373773 1loss/FBt3/assignment_loss‚0.18374991031193552 /loss/FRt2/detection_loss‚2.1352452754974367 /loss/FBt1/detection_loss‚3.3932648572054775 /loss/FBt4/detection_loss‚3.8088075280189515 epoch‚3 % _timestamp‚1.7884479635093873e+09 /loss/FRt1/detection_loss‚3.0587422231148027 .loss/FRt3/detection_loss‚2.008841513718168 1loss/SRqqt2/detection_loss‚2.0578879630565643 0loss/FRt3/assignment_loss‚2.4913122773170473 2loss/SRqqt2/assignment_loss‚1.3449529764056205 2loss/SRqqt3/assignment_loss‚1.0948221434609613 1loss/SRqqt3/detection_loss‚1.3702115714550018¼K`â“Æ‚(:ˆ æ™æÔ«íóü gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.978' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5956 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0R\Ç‚(:Ñ æ™æÔ”Áõþ network.sent‚ 8.453285e+06 disk.drbd2080.out‚0 disk.sda3.out‚ 117.69921875 disk.drbd2266.out‚0# disk.dm-0.out‚12377.77978515625& disk.nvme0n1.out‚12377.77197265625 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0$ memory_percent‚96.66407108306885* disk./.usagePercent‚20.149972080842346$ disk./.usageGB‚300.2491455078125 disk.sda.in‚0 disk.sda.out‚ 117.69921875 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 cpu‚25.35101356130743( disk.nvme0c0n1.out‚12377.77197265625 disk.rbd1.out‚ 16.3212890625 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.dm-1.in‚0 network.recv‚ 1.104659e+06 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.sda3.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.dm-1.out‚ 117.69921875( proc.memory.availableMB‚ 546.55859375# proc.memory.rssMB‚ 2570.11328125* proc.memory.percent‚15.686726570129395 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0S*~užÈ‚(:“ õ™æÔ÷Ñóü gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.295' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.458999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0èœòæZÉ‚(:Ï õ™æÔá®’ÿ$ disk./.usageGB‚300.2765197753906 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 16.3212890625& disk.nvme0n1.out‚12417.15087890625 disk.drbd1716.out‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚12417.15087890625 network.recv‚ 1.109893e+06 disk.sda2.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0$ memory_percent‚96.66256904602051# proc.memory.rssMB‚ 2570.11328125* disk./.usagePercent‚20.151809190908054 disk.dm-1.out‚ 118.21484375 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125# disk.dm-0.out‚12417.15087890625 disk.drbd1716.in‚0 cpu‚25.3494893475813 network.sent‚ 8.561033e+06 disk.dm-1.in‚0 disk.drbd2265.in‚0 disk.sda3.in‚0 disk.sda3.out‚ 118.21484375 disk.drbd2266.in‚0' proc.memory.availableMB‚ 546.8046875* proc.memory.percent‚15.686726570129395 disk.sda.out‚ 118.21484375 disk.sda1.in‚0 disk.sda1.out‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-0.in‚0.078125 proc.cpu.threads‚17ƒQ 2Ê- _wandbruntime‚3115  _runtime‚3115g…±ÇËÁ & lr-AdamW/pg2‚0.0001085224804707232 trainer/global_step‚3899 % _timestamp‚1.7884479997908082e+09 _runtime‚3115.262410838 _step‚317 & lr-AdamW/pg1‚0.0001085224804707232R•d-ÌÌÆ & lr-AdamW/pg1‚0.0001085224804707232 & lr-AdamW/pg2‚0.0001085224804707232 trainer/global_step‚3899 % _timestamp‚1.7884479997908082e+09 _runtime‚3115.262410838 _step‚317½xa2Í- _wandbruntime‚3115  _runtime‚3115@Õó?eÎÖO \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚220812313115648 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚73612359 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 $ _timestamp‚1.788447999791532e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 130324689 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚73612359 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚226754639436288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚230020323281920 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚56712332 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _step‚318 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚56712332 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚220812330155008 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚51801529 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚226754621433344 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3266358211072 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚56712330 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33090158 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3254332320174 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3253677882418 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚73632397 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56712330 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚230020997647360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3265701848576 NÕ]wE;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18710909 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3265701848576 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚226754639436288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚51801065 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚462 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3427197445632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚224066662475182 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚224065990998066 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 130344729 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33090156 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3427365263872 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18711371 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚230020323281920 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚230020997647360 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3427365263872 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18711371 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚230020323281920 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚230020997647360 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚56712330 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚230020997647360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040¾‡¸’á2Ð- _wandbruntime‚3115  _runtime‚3115i3¶ç&ÑáM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚460 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚73650877 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 _step‚319 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18715409 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚251106717665280 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3428045034496 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚224121711563590 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚224122383040914 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3255148755346 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚226810992475136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚230078170127360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51814195 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33098326 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3267177652224 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 130357159 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚73630813 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚230077495748608 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚220867234285568 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3266521276416 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚226810992475136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 130357159 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51813733 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3267177652224 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56726346 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130377225 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884480004734976e+09 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3428219130880 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚247678696570880 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚220867217246208 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3254494317382 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 130377225 trainer/global_step‚3899 FlÒæM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚51813733 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3267177652224 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚56726346 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 130377225 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884480004734976e+09 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3428219130880 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚247678696570880 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚220867217246208 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3254494317382 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 130377225 trainer/global_step‚3899 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18715409 XDDeviceStatsMonitor.on_train_batch_end/ å<)€ inactive_split_bytes.all.freed‚251106717665280 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3428045034496 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚224121711563590 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚224122383040914 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3255148755346 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚226810992475136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚230078170127360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚51814195 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33098326 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3267177652224 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 130357159 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚73630813 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚230077495748608 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚220867234285568 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3266521276416 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚226810992475136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 130357159 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚226852704342590 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚3949 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3295167659582 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57413132 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18938388 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚232878961030144 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚254169834364416 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚232879635395584 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3307344016896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131951541 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚223557519643648 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33498558 _runtime‚3152.030098866 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚254169653091840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57413132 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚74538411 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚229572291378688 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3307344016896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚232879635395584 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚229572273375744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 _step‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52436946 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚250699582716416 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚223557536683008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57413130 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚250699558776320 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3470251648000 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3470094315520 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33498556 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚232878961030144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3306687654400 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496‚Âzâ'áÜO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚57413130 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚250699558776320 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3470251648000 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3470094315520 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚33498556 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚232878961030144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3306687654400 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚18937961 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚226852704342590 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚3949 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3295167659582 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚57413132 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚18938388 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚232878961030144 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚254169834364416 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚232879635395584 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3307344016896 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 131951541 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚223557519643648 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚33498558 _runtime‚3152.030098866 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚254169653091840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚57413132 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚74538411 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚229572291378688 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3307344016896 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚232879635395584 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚229572273375744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 _step‚322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚52436946 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚250699582716416 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚223557536683008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952©pW_2â- _wandbruntime‚3152  _runtime‚3152ÛØXSrãàM REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18942508 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚223612423774208 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132004077 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3295984094754 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 3152.73419884 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 131984011 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3307507082240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52449232 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚229628626414592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 % _timestamp‚1.7884480372620032e+09 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57427148 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57427146 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚226907753430998 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚232936133496832 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3470952563200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3295329656790 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚254232349627904 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 132004077 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚232936807875584 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚74556865 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚229628644417536 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57427146 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚250761209513984 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 trainer/global_step‚3949 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 131984011 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚223612440813568 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3308163458048 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33506726 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚250761233454080 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33506724 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚254232162077184 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚74576929r…eÒë&äåM \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚254232349627904 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 132004077 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚232936807875584 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚74556865 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚229628644417536 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚57427146 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚250761209513984 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 trainer/global_step‚3949 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 131984011 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚223612440813568 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3308163458048 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33506726 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚250761233454080 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33506724 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚254232162077184 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚74576929 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚18942508 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚223612423774208 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 132004077 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3295984094754 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 3152.73419884 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 131984011 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3307507082240 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚52449232 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚229628626414592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 % _timestamp‚1.7884480372620032e+09 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚57427148 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚57427146 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚226907753430998 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚232936133496832 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3470952563200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3295329656790ÃR±=E2å- _wandbruntime‚3152  _runtime‚315250N0VæÐ .loss/FRt2/detection_loss‚2.023010887882926 /loss/FRt3/detection_loss‚1.7900674249231816 1loss/SRqqt3/detection_loss‚1.2403449671286526 /loss/FBt1/detection_loss‚2.9484996596972146 /loss/FBt2/detection_loss‚2.3558900356292725 /loss/FRt2/assignment_loss‚3.326217847791585 epoch‚3 _step‚324 0loss/FBt3/assignment_loss‚0.2547348363119957 -loss/FRt4/detection_loss‚2.27323311212517 0loss/SRqqt4/detection_loss‚1.504405001444476 %loss/total_loss‚46.70117318392664 2loss/SRqqt4/assignment_loss‚1.3258756844060762 0loss/FBt4/assignment_loss‚0.8040546242679868 trainer/global_step‚3949 /loss/FRt1/assignment_loss‚4.731025000413259 2loss/SRqqt3/assignment_loss‚0.9985958179176757 /loss/FBt2/assignment_loss‚0.543861842794043 1loss/SRqqt2/detection_loss‚1.8710947002151184 /loss/FBt4/detection_loss‚2.9118137189320157 0loss/FRt4/assignment_loss‚2.6455112979525612 2loss/SRqqt1/assignment_loss‚2.5266163125634193 0loss/SRqqt1/detection_loss‚2.657914200797677 .loss/FBt3/detection_loss‚0.862547995974975 % _timestamp‚1.7884480372638083e+09 /loss/FRt1/detection_loss‚3.1504490996400514 0loss/FRt3/assignment_loss‚1.7386102163791657 2loss/SRqqt2/assignment_loss‚1.3974966501284922 0loss/FBt1/assignment_loss‚0.8193022486096501 _runtime‚3152.735305568¡J |[çÕ 2loss/SRqqt4/assignment_loss‚1.3258756844060762 0loss/FBt4/assignment_loss‚0.8040546242679868 trainer/global_step‚3949 /loss/FRt1/assignment_loss‚4.731025000413259 2loss/SRqqt3/assignment_loss‚0.9985958179176757 /loss/FBt2/assignment_loss‚0.543861842794043 1loss/SRqqt2/detection_loss‚1.8710947002151184 /loss/FBt4/detection_loss‚2.9118137189320157 0loss/FRt4/assignment_loss‚2.6455112979525612 2loss/SRqqt1/assignment_loss‚2.5266163125634193 0loss/SRqqt1/detection_loss‚2.657914200797677 .loss/FBt3/detection_loss‚0.862547995974975 % _timestamp‚1.7884480372638083e+09 /loss/FRt1/detection_loss‚3.1504490996400514 0loss/FRt3/assignment_loss‚1.7386102163791657 2loss/SRqqt2/assignment_loss‚1.3974966501284922 0loss/FBt1/assignment_loss‚0.8193022486096501 _runtime‚3152.735305568 .loss/FRt2/detection_loss‚2.023010887882926 /loss/FRt3/detection_loss‚1.7900674249231816 1loss/SRqqt3/detection_loss‚1.2403449671286526 /loss/FBt1/detection_loss‚2.9484996596972146 /loss/FBt2/detection_loss‚2.3558900356292725 /loss/FRt2/assignment_loss‚3.326217847791585 epoch‚3 _step‚324 0loss/FBt3/assignment_loss‚0.2547348363119957 -loss/FRt4/detection_loss‚2.27323311212517 0loss/SRqqt4/detection_loss‚1.504405001444476 %loss/total_loss‚46.70117318392664Ä:¢Ì“è‚(:ˆ ±šæÔ¢‚öý gpu.0.gpu‚48.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.206' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.6412 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0n@|Ré‚(:Ç ±šæÔÖºûÿ# proc.memory.rssMB‚ 2570.11328125% disk.nvme0c0n1.out‚12570.91796875 disk.drbd2266.out‚0 disk.sda.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0* disk./.usagePercent‚20.153655005229925$ disk./.usageGB‚300.3040237426758 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.rbd1.in‚0 cpu‚25.343928411899597 disk.sda3.out‚ 120.89453125 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 16.626953125 disk.dm-1.in‚0 disk.drbd1716.in‚0' proc.memory.availableMB‚ 546.3515625 proc.cpu.threads‚17 network.recv‚ 1.129127e+06 disk.dm-0.out‚12570.91796875* proc.memory.percent‚15.686726570129395 network.sent‚ 8.687003e+06 disk.drbd2266.in‚0 disk.nvme0n1.in‚0.078125 disk.sda.out‚ 120.89453125# disk.nvme0n1.out‚12570.91796875 disk.dm-1.out‚ 120.89453125 disk.drbd1716.out‚0$ memory_percent‚96.66533470153809 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.drbd2265.in‚0Q 6“ê‚(:ˆ ÀšæÔ Îâü gpu.0.gpu‚29.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.047' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8094 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0슟;Të‚(:É ÀšæÔœê€ disk.rbd0.out‚0* proc.memory.percent‚15.686726570129395& disk.nvme0n1.out‚12627.80224609375 disk.drbd2266.in‚0( proc.memory.availableMB‚ 546.35546875 network.recv‚ 1.135941e+06 disk.sda2.in‚0 disk.drbd1716.out‚0$ memory_percent‚96.66531085968018 proc.cpu.threads‚17* disk./.usagePercent‚20.153318099314866$ disk./.usageGB‚300.2990036010742 disk.sda1.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 16.876953125 disk.dm-1.out‚ 121.359375 disk.sda3.in‚0 network.sent‚ 8.795762e+06 disk.sda.out‚ 121.359375 disk.sda3.out‚ 121.359375 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚12627.80224609375# proc.memory.rssMB‚ 2570.11328125 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚12627.80224609375 cpu‚25.3425543141954 disk.sda2.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0±A]Ó2ì- _wandbruntime‚3189  _runtime‚3189U^ÅÉíà ' lr-AdamW/pg1‚0.00010722197868387141 ' lr-AdamW/pg2‚0.00010722197868387141 trainer/global_step‚3999 % _timestamp‚1.7884480739125345e+09 _runtime‚3189.384154994 _step‚325Y@LÎîÈ _runtime‚3189.384154994 _step‚325 ' lr-AdamW/pg1‚0.00010722197868387141 ' lr-AdamW/pg2‚0.00010722197868387141 trainer/global_step‚3999 % _timestamp‚1.7884480739125345e+09ÅÅ6Ä™2ï- _wandbruntime‚3189  _runtime‚3189’Èñ¹•ð×O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3348329822720 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3512950415872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚75464463 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3347673460224 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53070874 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19164371 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚229638746209998 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚75484501 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚235737598778368 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚229638074732882 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚226302726171648 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 133578393 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58113930 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚235737598778368 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚232389943321088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 133598433 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3347673460224 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚75464463 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚3999 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚235738273143808 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚232389925318144 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚253782119599616 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚75484501 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚253782095659520 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3513120331264 =6DeviceStatsMonit¨ŠeH or.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7884480739132185e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19163918 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚235738273143808 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58113930 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚235737598778368 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚232389943321088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 133598433 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3347673460224 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚75464463 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚3999 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚235738273143808 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚232389925318144 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚253782119599616 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚75484501 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚253782095659520 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3513120331264 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7884480739132185e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19163918 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚235738273143808 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58127948 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3349149263872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚232446296359936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚232446296359936 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚253843746397184 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚235795445623808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133630929 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33915124 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚75482917 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19168466 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚75502981 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚257357551805440 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 trainer/global_step‚3999 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚232446278356992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3513971116032 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚226357630302208 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚257357741453312 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58127946 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 133610863 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚75482917 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3349149263872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53083590 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚226357647341568 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33915126 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚235794771245056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚253843770337280 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133630929 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58127946 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19168922Ä/š%ôæM PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚232446296359936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚232446296359936 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚253843746397184 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚235795445623808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 133630929 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚33915124 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚75482917 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19168466 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚75502981 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚257357551805440 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 trainer/global_step‚3999 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚232446278356992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3513971116032 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚226357630302208 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚257357741453312 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚58127946 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 133610863 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚75482917 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3349149263872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53083590 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚226357647341568 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚33915126 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚235794771245056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚253843770337280 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 133630929 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚58127946 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19168922 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 >8DeviceStatsMonitor.on_train_batch_end/num_oom_r!ðÔˆRejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚58127948 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3349149263872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0ÇN8üb2õ- _wandbruntime‚3190  _runtime‚3190å¤eÀZöÔ 0loss/FRt4/assignment_loss‚3.1375061786174774 1loss/SRqqt2/assignment_loss‚1.850533402739809 /loss/FRt3/detection_loss‚2.0522104531526564 0loss/SRqqt1/detection_loss‚2.866752564907074 1loss/SRqqt4/detection_loss‚1.6921389183354756 /loss/FBt4/detection_loss‚3.6504490673542023 _runtime‚3190.086470077 /loss/FRt1/assignment_loss‚4.087068726619084 /loss/FRt1/detection_loss‚3.1382021407286325 /loss/FRt2/detection_loss‚2.2974013487497964 /loss/FBt1/detection_loss‚2.8341981053352354 epoch‚3 2loss/SRqqt4/assignment_loss‚1.7474643676763488 1loss/FBt3/assignment_loss‚0.20634222050754578 0loss/FBt4/assignment_loss‚0.8114321589237079 1loss/SRqqt3/detection_loss‚1.3958619479732266 0loss/FRt2/assignment_loss‚3.3956133848742436 .loss/FBt2/detection_loss‚2.196503776311874 _step‚328 .loss/FBt3/detection_loss‚0.720300581674149 trainer/global_step‚3999 % _timestamp‚1.7884480746146562e+09 2loss/SRqqt3/assignment_loss‚1.1692662267977822 0loss/FBt2/assignment_loss‚0.7585205093026162 /loss/FRt4/detection_loss‚2.2705987319350243 /loss/SRqqt2/detection_loss‚1.71080708446411 &loss/total_loss‚49.440838239350754 1loss/SRqqt1/assignment_loss‚2.558570834994316 1loss/FBt1/assignment_loss‚0.32687533910370803 0loss/FRt3/assignment_loss‚2.5662201682726544F 'ä_÷Ù 0loss/FRt3/assignment_loss‚2.5662201682726544 0loss/FRt4/assignment_loss‚3.1375061786174774 1loss/SRqqt2/assignment_loss‚1.850533402739809 /loss/FRt3/detection_loss‚2.0522104531526564 0loss/SRqqt1/detection_loss‚2.866752564907074 1loss/SRqqt4/detection_loss‚1.6921389183354756 /loss/FBt4/detection_loss‚3.6504490673542023 _runtime‚3190.086470077 /loss/FRt1/assignment_loss‚4.087068726619084 /loss/FRt1/detection_loss‚3.1382021407286325 /loss/FRt2/detection_loss‚2.2974013487497964 /loss/FBt1/detection_loss‚2.8341981053352354 epoch‚3 2loss/SRqqt4/assignment_loss‚1.7474643676763488 1loss/FBt3/assignment_loss‚0.20634222050754578 0loss/FBt4/assignment_loss‚0.8114321589237079 1loss/SRqqt3/detection_loss‚1.3958619479732266 0loss/FRt2/assignment_loss‚3.3956133848742436 .loss/FBt2/detection_loss‚2.196503776311874 _step‚328 .loss/FBt3/detection_loss‚0.720300581674149 trainer/global_step‚3999 % _timestamp‚1.7884480746146562e+09 2loss/SRqqt3/assignment_loss‚1.1692662267977822 0loss/FBt2/assignment_loss‚0.7585205093026162 /loss/FRt4/detection_loss‚2.2705987319350243 /loss/SRqqt2/detection_loss‚1.71080708446411 &loss/total_loss‚49.440838239350754 1loss/SRqqt1/assignment_loss‚2.558570834994316 1loss/FBt1/assignment_loss‚0.32687533910370803È¿WÆV“ø‚(:ˆ ÏšæÔ®ò“ý gpu.0.gpu‚34.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚167.056' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.4112 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0°­î«Nù‚(:à ϚæÔŽÈ‹€ disk.rbd0.in‚0# disk.dm-0.out‚12663.85693359375 disk.sda.out‚ 121.78125% proc.memory.availableMB‚ 546.28125 cpu‚25.34119172175833$ disk./.usageGB‚300.3110542297363 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚12663.85693359375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 network.recv‚ 1.13898e+06 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.dm-0.in‚0.078125 proc.cpu.threads‚17 disk.sda2.in‚0 disk.dm-1.out‚ 121.78125 disk.drbd2080.in‚0# proc.memory.rssMB‚ 2570.11328125* disk./.usagePercent‚20.154126827115533& disk.nvme0n1.out‚12663.85693359375 disk.dm-1.in‚0 network.sent‚ 8.800742e+06 disk.sda2.out‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 121.78125 disk.sda.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 17.001953125$ memory_percent‚96.66576385498047* proc.memory.percent‚15.686726570129395Ã};žú‚(:“ ÞšæÔØÔý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚142.147' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.429399999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ã…½­Kû‚(:À ÞšæÔþ§Žÿ network.sent‚ 8.909265e+06 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda.in‚0 disk.sda2.out‚0 cpu‚25.339859078157577 disk.sda1.in‚0 disk.sda1.out‚0% disk./.usageGB‚300.31463623046875 disk.dm-0.in‚0.078125 disk.dm-0.out‚ 12703.46875 disk.rbd1.in‚0 disk.rbd1.out‚ 17.064453125 disk.nvme0n1.out‚ 12703.46875( proc.memory.availableMB‚ 546.52734375* proc.memory.percent‚15.686726570129395 disk.drbd2080.out‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.sda.out‚ 122.39453125 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125$ memory_percent‚96.66426181793213 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.dm-1.out‚ 122.39453125" disk.nvme0c0n1.out‚ 12703.46875 network.recv‚ 1.146172e+06* disk./.usagePercent‚20.154367218190185 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.dm-1.in‚0# proc.memory.rssMB‚ 2570.11328125 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.sda3.out‚ 122.394531256™'—“ü‚(:ˆ íšæÔÉɨý gpu.0.gpu‚28.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.482' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8964 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0/ŽFý‚(:» íšæÔ’ôÿ disk.sda1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0* proc.memory.percent‚15.686726570129395 proc.cpu.threads‚17 disk.sda2.out‚0% disk./.usageGB‚300.32123947143555 disk.rbd0.in‚0 disk.dm-0.out‚ 12743.796875 disk.sda.out‚ 122.859375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚0# disk.nvme0c0n1.out‚ 12743.796875 network.sent‚ 8.914512e+06 disk.drbd1716.in‚0 disk.drbd2080.out‚0# proc.memory.rssMB‚ 2570.11328125 cpu‚25.33852281604937 disk.sda.in‚0$ memory_percent‚96.66428565979004' proc.memory.availableMB‚ 546.5234375 disk.sda1.out‚0 disk.rbd1.out‚ 17.064453125 disk.dm-1.out‚ 122.859375 disk.sda2.in‚0* disk./.usagePercent‚20.154810367231953 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125! disk.nvme0n1.out‚ 12743.796875 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.sda3.out‚ 122.859375 network.recv‚ 1.149493e+06ƒ[äb2þ- _wandbruntime‚3226  _runtime‚3226&Æ Èÿ ' lr-AdamW/pg1‚0.00010655794303315473 ' lr-AdamW/pg2‚0.00010655794303315473 trainer/global_step‚4049 % _timestamp‚1.7884481110382729e+09 _runtime‚ 3226.50988089 _step‚329X”xÍÍ€Ç % _timestamp‚1.7884481110382729e+09 _runtime‚ 3226.50988089 _step‚329 ' lr-AdamW/pg1‚0.00010655794303315473 ' lr-AdamW/pg2‚0.00010655794303315473 trainer/global_step‚4049ÉK™2- _wandbruntime‚3226  _runtime‚3226ÍÕ® Ý'‚×O UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34315358 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚76410553 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚76410553 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53706155 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34315356 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135225285 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3388659266048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19390799 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚4049 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚229047932699648 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3376838338398 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3555908018688 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3389315628544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3388659266048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚238596236526592 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3226.510866189 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚235207595263488 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53706604 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚256864632542720 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3556075836928 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚238596910892032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚235207577260544 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚232424788077406 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3376183900642 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3389315628544 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58814732 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚232424116600290 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 % _timestamp‚1.7884481110390875e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚238596910892032 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚238596236526592 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚235207577260544 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚260420540561408 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58814732 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚260420732319744 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚76390515 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58814730 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚235207595263488 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19391246 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚447 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚76390515 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414Ù†?â'ƒÜO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚229047932699648 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3376838338398 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3555908018688 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3389315628544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3388659266048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚238596236526592 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3226.510866189 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚235207595263488 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚53706604 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚256864632542720 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3556075836928 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚238596910892032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚235207577260544 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚232424788077406 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3376183900642 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3389315628544 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚58814732 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚232424116600290 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 % _timestamp‚1.7884481110390875e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚238596910892032 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚238596236526592 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚235207577260544 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚260420540561408 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚58814732 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚260420732319744 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚76390515 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚58814730 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚235207595263488 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19391246 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚447 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚76390515 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34315358 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚76410553 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚76410553 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚53706155 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚34315356 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 135225285 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3388659266048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19390799 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚4049 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚235263930299392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3390135069696 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3556761640960 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 135237715 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚238654083372032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚76408969 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34323524 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚76429033 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚232480508643138 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚235263948302336 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚232479837165814 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚235263930299392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53718911 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135257781 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53719364 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19395838 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚260483238763520 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚229102836830208 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3377654773570 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _runtime‚3227.210063539 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 135237715 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚238654083372032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚76408969 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚34323524 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚76429033 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚232480508643138 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚235263948302336 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚232479837165814 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚235263930299392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚53718911 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 135257781 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚53719364 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19395838 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚260483238763520 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚229102836830208 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3377654773570 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _runtime‚3227.210063539 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚235263930299392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3390135069696 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3556761640960 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088ËsçW2‡- _wandbruntime‚3227  _runtime‚3227.ÜPZˆÔ /loss/FRt1/assignment_loss‚4.402123388571617 0loss/FBt2/assignment_loss‚0.5427932573707452 epoch‚3 _runtime‚3227.211210163 _step‚332 2loss/SRqqt2/assignment_loss‚1.8779295136195582 /loss/FBt3/detection_loss‚0.7545850745518692 0loss/FRt3/assignment_loss‚1.9838986160620204 2loss/SRqqt3/assignment_loss‚1.3054932628703468 0loss/FBt1/assignment_loss‚0.7407629612416713 1loss/SRqqt2/detection_loss‚2.3737291806452983 0loss/SRqqt3/detection_loss‚1.129985429236994 .loss/FBt2/detection_loss‚2.951322772286155 1loss/FBt3/assignment_loss‚0.21274785186392303 /loss/FBt4/detection_loss‚3.0283613758427754 /loss/FRt2/assignment_loss‚3.750465433772017 .loss/FRt3/detection_loss‚1.914847270497736 1loss/SRqqt1/detection_loss‚2.4829702209681273 1loss/SRqqt4/detection_loss‚1.7047398908571763 trainer/global_step‚4049 % _timestamp‚1.7884481117397285e+09 1loss/SRqqt1/assignment_loss‚2.447029357776046 0loss/FBt4/assignment_loss‚0.7418977628861155 /loss/FBt1/detection_loss‚2.8233366516920237 &loss/total_loss‚49.576978362890685 .loss/FRt2/detection_loss‚2.233104300208208 /loss/FRt4/detection_loss‚2.4177500569820403 0loss/FRt4/assignment_loss‚3.3058652436733245 2loss/SRqqt4/assignment_loss‚1.7230542208660733 /loss/FRt1/detection_loss‚2.7281852685488186^½ò_‰Ù epoch‚3 _runtime‚3227.211210163 _step‚332 2loss/SRqqt2/assignment_loss‚1.8779295136195582 /loss/FBt3/detection_loss‚0.7545850745518692 0loss/FRt3/assignment_loss‚1.9838986160620204 2loss/SRqqt3/assignment_loss‚1.3054932628703468 0loss/FBt1/assignment_loss‚0.7407629612416713 1loss/SRqqt2/detection_loss‚2.3737291806452983 0loss/SRqqt3/detection_loss‚1.129985429236994 .loss/FBt2/detection_loss‚2.951322772286155 1loss/FBt3/assignment_loss‚0.21274785186392303 /loss/FBt4/detection_loss‚3.0283613758427754 /loss/FRt2/assignment_loss‚3.750465433772017 .loss/FRt3/detection_loss‚1.914847270497736 1loss/SRqqt1/detection_loss‚2.4829702209681273 1loss/SRqqt4/detection_loss‚1.7047398908571763 trainer/global_step‚4049 % _timestamp‚1.7884481117397285e+09 1loss/SRqqt1/assignment_loss‚2.447029357776046 0loss/FBt4/assignment_loss‚0.7418977628861155 /loss/FBt1/detection_loss‚2.8233366516920237 &loss/total_loss‚49.576978362890685 .loss/FRt2/detection_loss‚2.233104300208208 /loss/FRt4/detection_loss‚2.4177500569820403 0loss/FRt4/assignment_loss‚3.3058652436733245 2loss/SRqqt4/assignment_loss‚1.7230542208660733 /loss/FRt1/detection_loss‚2.7281852685488186 /loss/FRt1/assignment_loss‚4.402123388571617 0loss/FBt2/assignment_loss‚0.5427932573707452ÌwKÄp‘Š‚(:† üšæÔ­“úü gpu.0.gpu‚30.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚181.94' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.388 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0’¦stV‹‚(:Ë üšæÔÔÜüþ disk.drbd2266.in‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 123.54296875 disk.sda3.out‚ 123.54296875 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.rbd1.out‚ 17.0966796875 disk.nvme0n1.in‚0.078125$ disk.nvme0n1.out‚12787.642578125 disk.sda3.in‚0* proc.memory.percent‚15.686726570129395 disk.sda1.out‚0 disk.dm-1.in‚0 disk.drbd2265.in‚0( proc.memory.availableMB‚ 546.48046875# proc.memory.rssMB‚ 2570.11328125 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125& disk.nvme0c0n1.out‚12787.642578125 disk.drbd2266.out‚0 network.sent‚ 8.920951e+06 disk.dm-0.in‚0.078125 disk.sda.out‚ 123.54296875 cpu‚25.33727793741173 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.drbd2265.out‚0 network.recv‚ 1.154397e+06! disk.dm-0.out‚12787.642578125$ memory_percent‚96.66454792022705 proc.cpu.threads‚17* disk./.usagePercent‚20.154983684332635$ disk./.usageGB‚300.3238220214844 disk.sda.in‚0 disk.drbd1716.out‚0LØB-Œ‚(:’ ‹›æÔèáõü gpu.0.gpu‚49.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.652' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚36.53039999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0²î"ÚG‚(:¼ ‹›æÔ±¦Öþ disk.drbd2080.in‚0 disk.sda3.out‚124.0625 disk.rbd0.in‚0 disk.sda.out‚124.0625* proc.memory.percent‚15.686726570129395$ disk./.usageGB‚300.3354797363281 disk.sda2.out‚0 disk.drbd2265.out‚0$ memory_percent‚96.66461944580078* disk./.usagePercent‚20.155766043357275 disk.drbd2265.in‚0% proc.memory.availableMB‚ 546.46875 disk.rbd1.out‚ 17.2060546875 disk.dm-1.in‚0 disk.dm-1.out‚124.0625& disk.nvme0c0n1.out‚12825.380859375 disk.drbd2266.out‚0 disk.rbd0.out‚0# proc.memory.rssMB‚ 2570.11328125 proc.cpu.threads‚17 disk.sda1.in‚0 disk.rbd1.in‚0$ disk.nvme0n1.out‚12825.380859375 disk.sda.in‚0 network.recv‚ 1.159723e+06 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 cpu‚25.33589449558114 network.sent‚ 9.028686e+06 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd1716.out‚0! disk.dm-0.out‚12825.380859375Ký2Ž- _wandbruntime‚3263  _runtime‚3263Ú–CAÇÁ ' lr-AdamW/pg1‚0.00010588490142106734 ' lr-AdamW/pg2‚0.00010588490142106734 trainer/global_step‚4099 # _timestamp‚1.78844814774931e+09 _runtime‚3263.221103762 _step‚333¡)‡ŽÌÆ ' lr-AdamW/pg1‚0.00010588490142106734 ' lr-AdamW/pg2‚0.00010588490142106734 trainer/global_step‚4099 # _timestamp‚1.78844814774931e+09 _runtime‚3263.221103762 _step‚333Í;Ï,2‘- _wandbruntime‚3263  _runtime‚3263÷(´Ý'’×O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚59515532 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚238025229202944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3429645071872 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3430301434368 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚77316567 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚34723758 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚238025247205888 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 136852137 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54341191 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚241454874274816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚235210158467698 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚334 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚263546092050432 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3417673677806 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19616998 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚241455548640256 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59515532 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚435 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚259947193366016 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3598737157632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚238025247205888 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3598898684416 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3430301434368 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54340754 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚259947169425920 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚235210829944814 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19617433 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDeviceStatsMonitor.on_train_batch_start/acÿ¸ü„î&tive.small_pool.peak‚24128 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚19616998 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚241455548640256 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚59515532 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚435 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚259947193366016 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3598737157632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚238025247205888 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3598898684416 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3430301434368 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54340754 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚259947169425920 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚235210829944814 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚19617433 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚241454874274816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚235210158467698 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚334Î…¢dó2”- _wandbruntime‚3263  _runtime‚3263#N/ýç&•áM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚59529546 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3431120875520 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚77335021 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59529548 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 136864567 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59529548 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚260008820163584 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚238081600244736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3430464499712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚231848060397568 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚231848043358208 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 202230784 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚241512721120256 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3431120875520 _runtime‚3263.898021278 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚260008844103680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 136864567 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54353951 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19622025 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚77335021 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚238081600244736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34731926 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3599573751808 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚241512046741504 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59529546 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54353508 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19621584 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 178290688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚77355085 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚241512721120256 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚263608393915392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884481484256709e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136884633 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚443 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚441 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚4099 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚235266550510546 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 136884633 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚443 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚441 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚4099 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚235266550510546 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚59529548 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 136864567 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚59529548 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚260008820163584 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚238081600244736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3430464499712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚231848060397568 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚231848043358208 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 202230784 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚241512721120256 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3431120875520 _runtime‚3263.898021278 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚260008844103680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 136864567 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54353951 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19622025 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚77335021 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚238081600244736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚34731926 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3599573751808 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚241512046741504 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚59529546 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54353508 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19621584 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 178290688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚77355085 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚241512721120256 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚263608393915392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884481484256709e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401ÏñÕý¹2—- _wandbruntime‚3263  _runtime‚3263Kß®È]˜× 1loss/SRqqt3/detection_loss‚1.2763871126682371 /loss/FBt3/detection_loss‚0.8415876129183216 _step‚336 2loss/SRqqt2/assignment_loss‚1.5956302704289556 2loss/SRqqt3/assignment_loss‚1.2191604754912693 /loss/FRt3/detection_loss‚2.0699374033345115 /loss/FRt4/detection_loss‚2.2859182709455492 0loss/SRqqt4/detection_loss‚1.562809448922053 0loss/FBt4/assignment_loss‚0.9195982466141382 0loss/FBt2/assignment_loss‚0.4095132694875474 1loss/SRqqt1/detection_loss‚2.8991534445020886 .loss/FBt2/detection_loss‚2.758219711482525 % _timestamp‚1.7884481484275353e+09 trainer/global_step‚4099 1loss/FBt1/assignment_loss‚0.45297650855015736 .loss/FRt2/detection_loss‚2.190062338555301 /loss/FBt1/detection_loss‚3.0462309176271614 &loss/total_loss‚50.987039506607914 _runtime‚3263.899067873 /loss/FRt1/assignment_loss‚4.790109896659851 /loss/FBt4/detection_loss‚4.6055677731831866 epoch‚3 0loss/FRt2/assignment_loss‚3.9523473746246762 /loss/FRt4/assignment_loss‚3.076462163925171 2loss/SRqqt1/assignment_loss‚2.2729361918237476 2loss/SRqqt4/assignment_loss‚1.3287108947697561 0loss/FRt3/assignment_loss‚1.7922076251771715 1loss/FBt3/assignment_loss‚0.23980685587504308 /loss/FRt1/detection_loss‚3.6181257516145706 1loss/SRqqt2/detection_loss‚1.7835799474269152-áÏM™Ü .loss/FBt2/detection_loss‚2.758219711482525 % _timestamp‚1.7884481484275353e+09 trainer/global_step‚4099 1loss/FBt1/assignment_loss‚0.45297650855015736 .loss/FRt2/detection_loss‚2.190062338555301 /loss/FBt1/detection_loss‚3.0462309176271614 &loss/total_loss‚50.987039506607914 _runtime‚3263.899067873 /loss/FRt1/assignment_loss‚4.790109896659851 /loss/FBt4/detection_loss‚4.6055677731831866 epoch‚3 0loss/FRt2/assignment_loss‚3.9523473746246762 /loss/FRt4/assignment_loss‚3.076462163925171 2loss/SRqqt1/assignment_loss‚2.2729361918237476 2loss/SRqqt4/assignment_loss‚1.3287108947697561 0loss/FRt3/assignment_loss‚1.7922076251771715 1loss/FBt3/assignment_loss‚0.23980685587504308 /loss/FRt1/detection_loss‚3.6181257516145706 1loss/SRqqt2/detection_loss‚1.7835799474269152 1loss/SRqqt3/detection_loss‚1.2763871126682371 /loss/FBt3/detection_loss‚0.8415876129183216 _step‚336 2loss/SRqqt2/assignment_loss‚1.5956302704289556 2loss/SRqqt3/assignment_loss‚1.2191604754912693 /loss/FRt3/detection_loss‚2.0699374033345115 /loss/FRt4/detection_loss‚2.2859182709455492 0loss/SRqqt4/detection_loss‚1.562809448922053 0loss/FBt4/assignment_loss‚0.9195982466141382 0loss/FBt2/assignment_loss‚0.4095132694875474 1loss/SRqqt1/detection_loss‚2.Ù÷ïÎ8991534445020886Ð^”é“š‚(:ˆ š›æÔ¦Ùþ gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.766' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5532 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ñ"`îG›‚(:¼ š›æÔâçûÿ disk.drbd1716.out‚0 disk.sda.out‚124.75 disk.drbd2266.out‚0 disk.sda3.out‚124.75 disk.rbd0.out‚0 disk.dm-0.in‚0.078125$ memory_percent‚96.66507244110107( proc.memory.availableMB‚ 546.39453125# proc.memory.rssMB‚ 2570.11328125 network.recv‚ 1.165104e+06 disk.nvme0n1.in‚0.078125% disk./.usageGB‚300.34069061279297 disk.sda1.in‚0 proc.cpu.threads‚17 disk.drbd2266.in‚0 network.sent‚ 9.0354e+06 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.dm-1.in‚0' disk.nvme0c0n1.out‚12866.3330078125" disk.dm-0.out‚12866.3330078125% disk.nvme0n1.out‚12866.3330078125 disk.drbd1716.in‚0 cpu‚25.334498090011845* disk./.usagePercent‚20.156115749649047 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0 disk.rbd1.out‚ 17.2685546875 disk.dm-1.out‚124.75* proc.memory.percent‚15.686726570129395 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.rbd0.in‚0 disk.sda.in‚0 disk.sda1.out‚0 disk.sda2.out‚0”é]‘œ‚(:† ©›æÔãçŠý gpu.0.gpu‚40.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.38' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.476 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0fÌ7•X‚(:Í ©›æÔŠÕœ€ disk.sda3.in‚0 disk.rbd0.out‚0 disk.dm-1.out‚ 125.32421875 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0" disk.dm-0.out‚12904.7353515625* proc.memory.percent‚15.686726570129395 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 17.2685546875 disk.sda3.out‚ 125.32421875 disk.dm-0.in‚0.078125$ disk./.usageGB‚300.3603630065918 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 125.32421875 cpu‚25.33320665446626 disk.sda2.in‚0% disk.nvme0n1.out‚12904.7353515625 disk.drbd2080.in‚0 disk.drbd2080.out‚0$ memory_percent‚96.66726589202881 network.sent‚ 9.142932e+06 network.recv‚ 1.169794e+06) disk./.usagePercent‚20.15743598050312 disk.sda1.in‚0 disk.dm-1.in‚0' disk.nvme0c0n1.out‚12904.7353515625# proc.memory.rssMB‚ 2570.11328125 disk.sda2.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda.in‚0( proc.memory.availableMB‚ 546.03515625 proc.cpu.threads‚17=Më2ž- _wandbruntime‚3299  _runtime‚3299΄˜‹ÇŸÁ ' lr-AdamW/pg1‚0.00010520299410847243 ' lr-AdamW/pg2‚0.00010520299410847243 trainer/global_step‚4149 % _timestamp‚1.7884481843184478e+09 _runtime‚ 3299.7900532 _step‚3377pùÌ Æ _step‚337 ' lr-AdamW/pg1‚0.00010520299410847243 ' lr-AdamW/pg2‚0.00010520299410847243 trainer/global_step‚4149 % _timestamp‚1.7884481843184478e+09 _runtime‚ 3299.7900532у3"2¡- _wandbruntime‚3299  _runtime‚3299ur´AÝ'¢×O _runtime‚3299.790991601 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3470630877696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚244313512023040 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3471287240192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚78242619 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3641629117952 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚237996871812222 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60216332 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚240842881145344 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚240842899148288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 trainer/global_step‚4149 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚244314186388480 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚244313512023040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚266671518796800 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚237996200335106 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3471287240192 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3457854579458 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚78262657 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 138458949 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚234538345755648 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚429 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60216330 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚78242619 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54974969 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3641788547584 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 _step‚338 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3458509017214 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚244314186388480 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3470630877696 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54974538 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35132156 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3457854579458 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚78262657 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 138458949 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚234538345755648 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚429 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60216330 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚78242619 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚54974969 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3641788547584 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 _step‚338 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3458509017214 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚244314186388480 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3470630877696 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚54974538 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35132156 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚244313512023040 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3471287240192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚78242619 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3641629117952 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚237996871812222 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60216332 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚240842881145344 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚240842899148288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 trainer/global_step‚4149 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚244314186388480 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚244313512023040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚266671518796800 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚237996200335106 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3471287240192 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3471450305536 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚78261073 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35140326 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚78261073 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35140324 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60230346 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3472106681344 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚78281137 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19847360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60230346 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 138491419 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚240899252187136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54987686 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3471450305536 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚266734042327040 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚234593266925568 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19846944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚240899234184192 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚4149 % _timestamp‚1.7884481850030801e+09 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚416 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 _step‚339 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚244371358868480 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60230348 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 170773504 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138511485 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚3300.475238971 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138511485 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚263091380986880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3459325452386 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚78281137 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚234593249886208 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60230348 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚238051920900630 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3458671014422 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactiveÙ¯oö _split_bytes.small_pool.allocated‚ 3642661340160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚266733871553536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3642514506752 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚244370684489728 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3472106681344 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 138491419 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 146833408 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚418 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54987268 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚263091357046784Žƒµ«ì&¨æM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚416 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 _step‚339 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚244371358868480 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚60230348 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 170773504 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 138511485 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚3300.475238971 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 138511485 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚263091380986880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3459325452386 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚78281137 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚234593249886208 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60230348 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚238051920900630 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3458671014422 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3642661340160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚266733871553536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3642514506752 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚244370684489728 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3472106681344 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 138491419 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 146833408 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚418 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚54987268 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚263091357046784 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3471450305536 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚78261073 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35140326 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚78261073 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35140324 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60230346 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3472106681344 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚78281137 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚19847360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚60230346 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 138491419 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚240899252187136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚54987686 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3471450305536 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚266734042327040 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚234593266925568 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚19846944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚240899234184192 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚4149 % _timestamp‚1.7884481850030801e+09 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872Ó„´2©- _wandbruntime‚3300  _runtime‚3300c4oZªÔ trainer/global_step‚4149 0loss/FBt1/assignment_loss‚0.5383163395575442 .loss/FRt1/detection_loss‚2.906066060066223 /loss/FRt1/assignment_loss‚5.125775434754112 0loss/FRt2/assignment_loss‚3.7584629213809966 2loss/SRqqt2/assignment_loss‚1.9990118603137406 .loss/FBt4/detection_loss‚3.873032534122467 % _timestamp‚1.7884481850048494e+09 0loss/FRt3/assignment_loss‚2.4178412381340477 1loss/SRqqt4/detection_loss‚1.3461895459890365 _step‚340 .loss/FBt2/detection_loss‚2.516690170392394 _runtime‚3300.476336382 1loss/SRqqt1/assignment_loss‚2.590203814448849 1loss/FBt3/assignment_loss‚0.21532350328982833 .loss/FRt3/detection_loss‚2.178006411183114 1loss/SRqqt1/detection_loss‚2.7817452415343253 0loss/SRqqt2/detection_loss‚2.124149073931304 /loss/FBt3/detection_loss‚0.8378141328293298 epoch‚3 0loss/FBt2/assignment_loss‚0.7600620853387738 /loss/FRt4/detection_loss‚2.0133349833866725 1loss/FBt4/assignment_loss‚0.32764652890218715 1loss/SRqqt3/detection_loss‚1.4059815317139788 /loss/FBt1/detection_loss‚3.8284183541933694 /loss/FRt4/assignment_loss‚2.439023412190951 2loss/SRqqt3/assignment_loss‚1.2715207905334942 2loss/SRqqt4/assignment_loss‚1.7373911039158703 /loss/FRt2/detection_loss‚2.0751431715488433 &loss/total_loss‚51.067150243651454ë0Ž_«Ù 0loss/FRt2/assignment_loss‚3.7584629213809966 2loss/SRqqt2/assignment_loss‚1.9990118603137406 .loss/FBt4/detection_loss‚3.873032534122467 % _timestamp‚1.7884481850048494e+09 0loss/FRt3/assignment_loss‚2.4178412381340477 1loss/SRqqt4/detection_loss‚1.3461895459890365 _step‚340 .loss/FBt2/detection_loss‚2.516690170392394 _runtime‚3300.476336382 1loss/SRqqt1/assignment_loss‚2.590203814448849 1loss/FBt3/assignment_loss‚0.21532350328982833 .loss/FRt3/detection_loss‚2.178006411183114 1loss/SRqqt1/detection_loss‚2.7817452415343253 0loss/SRqqt2/detection_loss‚2.124149073931304 /loss/FBt3/detection_loss‚0.8378141328293298 epoch‚3 0loss/FBt2/assignment_loss‚0.7600620853387738 /loss/FRt4/detection_loss‚2.0133349833866725 1loss/FBt4/assignment_loss‚0.32764652890218715 1loss/SRqqt3/detection_loss‚1.4059815317139788 /loss/FBt1/detection_loss‚3.8284183541933694 /loss/FRt4/assignment_loss‚2.439023412190951 2loss/SRqqt3/assignment_loss‚1.2715207905334942 2loss/SRqqt4/assignment_loss‚1.7373911039158703 /loss/FRt2/detection_loss‚2.0751431715488433 &loss/total_loss‚51.067150243651454 trainer/global_step‚4149 0loss/FBt1/assignment_loss‚0.5383163395575442 .loss/FRt1/detection_loss‚2.906066060066223 /loss/FRt1/assignment_loss‚5.125775434754112Ô‘W2“¬‚(:ˆ Ç›æÔíçŽý gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.307' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.2614 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“»¥¬W­‚(:Ì Ç›æÔŪ–ÿ disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.sda.in‚0 cpu‚25.330682450393663' disk.nvme0c0n1.out‚12991.6611328125 disk.sda3.out‚ 126.33203125 disk.sda2.out‚0% disk.nvme0n1.out‚12991.6611328125 disk.drbd2265.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 126.33203125 disk.drbd1716.in‚0 disk.drbd2266.in‚0$ memory_percent‚96.66752815246582# proc.memory.rssMB‚ 2570.11328125 network.sent‚ 9.24557e+06 network.recv‚ 1.179156e+06 disk.sda.out‚ 126.33203125* proc.memory.percent‚15.686726570129395 proc.cpu.threads‚17 disk.dm-0.in‚0.078125" disk.dm-0.out‚12991.6611328125 disk.rbd1.out‚ 17.6123046875 disk.drbd1716.out‚0 disk.drbd2080.out‚0) disk./.usagePercent‚20.15893132051062$ disk./.usageGB‚300.3826446533203 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.drbd2266.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.rbd0.out‚0' proc.memory.availableMB‚ 545.9921875 disk.sda2.in‚0 disk.dm-1.in‚0½çzž®‚(:“ Ö›æÔÀ€Œý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚139.558' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.911599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0GÌÔgD¯‚(:¹ Ö›æÔÏÕþþ disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.drbd1716.out‚0$ memory_percent‚96.66752815246582# proc.memory.rssMB‚ 2570.11328125 disk.sda3.in‚0 disk.dm-0.out‚ 13056.125 disk.sda.out‚ 126.95703125 disk.drbd1716.in‚0 disk.nvme0n1.out‚ 13056.125 disk.dm-1.out‚ 126.95703125 proc.cpu.threads‚17 network.sent‚ 9.344305e+06 disk.sda2.in‚0 disk.rbd1.out‚ 17.6748046875 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd2265.in‚0% disk./.usageGB‚300.40796279907227 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125' proc.memory.availableMB‚ 545.9921875 cpu‚25.329432111075356 network.recv‚ 1.18617e+06* disk./.usagePercent‚20.160630442515373 disk.nvme0c0n1.out‚ 13056.125 disk.rbd0.out‚0 disk.sda.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0* proc.memory.percent‚15.686726570129395 disk.sda3.out‚ 126.95703125¶óÇô2°- _wandbruntime‚3336  _runtime‚3336‹yXÊDZÁ _runtime‚3336.661368565 _step‚341 & lr-AdamW/pg1‚0.0001045123632038319 & lr-AdamW/pg2‚0.0001045123632038319 trainer/global_step‚4199 % _timestamp‚1.7884482211897573e+09DåyEÌ²Æ % _timestamp‚1.7884482211897573e+09 _runtime‚3336.661368565 _step‚341 & lr-AdamW/pg1‚0.0001045123632038319 & lr-AdamW/pg2‚0.0001045123632038319 trainer/global_step‚4199Õ„^¾2³- _wandbruntime‚3336  _runtime‚3336 ñ‹öÝ'´×O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚266112243192320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3684566437376 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚79188709 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚247172149771264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚79188709 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚240782913679630 _step‚342 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3511616683520 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚269796833569792 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3498689918866 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚247172824136704 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3511616683520 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚60917130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚269796629228544 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _runtime‚3336.662329198 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚79168671 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚240782242202514 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 140085801 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3512273046016 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚79168671 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20068109 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60917130 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚458 trainer/global_step‚4199 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55609125 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚247172824136704 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚243660551090688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35540558 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚237283569323008 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚243660533087744 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55608665 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 140105841 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚243660533087744 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3499344356622 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60917132 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚243660551090688 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚247172149771264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20068567 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚237283569323008 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚243660533087744 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚55608665 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 140105841 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚243660533087744 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3499344356622 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚60917132 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚243660551090688 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚247172149771264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.alloc_A€‚¸ated‚20068567 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚79188709 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚247172149771264 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚79188709 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚240782913679630 _step‚342 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3511616683520 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚269796833569792 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3498689918866 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚247172824136704 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3511616683520 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚60917130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚269796629228544 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _runtime‚3336.662329198 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚79168671 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚240782242202514 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 140085801 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3512273046016 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚79168671 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20068109 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚60917130 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚458 trainer/global_step‚4199 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚55609125 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚247172824136704 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚243660551090688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35540558 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0ÖŽ•s2¶- _wandbruntime‚3337  _runtime‚3337“^ê³ç&·áM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚247229996616704 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚79187125 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3512436111360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚79187125 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55621756 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚243716886126592 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60931146 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3499506353830 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140118271 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚79207189 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3500160791794 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚4199 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚237338456414208 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚243716904129536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚237338473453568 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3513092487168 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140138337 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚266173917870080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140138337 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35548726 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚243716886126592 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 140118271 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55621325 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚240838634245362 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60931148 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚243716904129536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 _step‚343 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7884482218790932e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20073030 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844¶˜=ºì&¸æM KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20073030 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚247229996616704 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚79187125 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3512436111360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚79187125 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚55621756 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚243716886126592 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚60931146 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3499506353830 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 140118271 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚79207189 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3500160791794 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚4199 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚237338456414208 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚243716904129536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚237338473453568 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3513092487168 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 140138337 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚266173917870080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 140138337 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35548726 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚243716886126592 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 140118271 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚55621325 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚240838634245362 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚60931148 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚243716904129536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 _step‚343 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7884482218790932e+09×kºÈ2¹- _wandbruntime‚3337  _runtime‚3337í…xªZºÔ epoch‚3 /loss/FRt2/assignment_loss‚3.732748561717094 0loss/FRt4/assignment_loss‚3.1919942882325913 0loss/FBt1/assignment_loss‚0.6363084100355226 1loss/SRqqt2/detection_loss‚1.6932117567986857 0loss/SRqqt4/detection_loss‚1.733266646663348 trainer/global_step‚4199 1loss/SRqqt1/detection_loss‚2.8094711179534593 1loss/SRqqt3/detection_loss‚1.5947117693722248 /loss/FBt2/detection_loss‚1.8880320649880629 _runtime‚3337.352381398 /loss/FRt1/assignment_loss‚5.067676197737455 2loss/SRqqt4/assignment_loss‚1.3590598597570702 0loss/FBt2/assignment_loss‚0.8217053161136462 .loss/FRt4/detection_loss‚2.301483518216345 /loss/FBt3/detection_loss‚0.9013022438855842 % _timestamp‚1.7884482218808846e+09 _step‚344 1loss/SRqqt1/assignment_loss‚2.559399309257666 2loss/SRqqt2/assignment_loss‚1.4877667814797284 /loss/FRt1/detection_loss‚2.8756379410624504 /loss/FRt3/detection_loss‚1.8763163623944767 /loss/FBt1/detection_loss‚2.5296903523531826 0loss/FRt3/assignment_loss‚1.8795100697931253 2loss/SRqqt3/assignment_loss‚1.3007558591517747 /loss/FRt2/detection_loss‚2.3160758874517806 /loss/FBt4/detection_loss‚3.3732869148254396 %loss/total_loss‚48.64978924612436 0loss/FBt3/assignment_loss‚0.2176822180299312 0loss/FBt4/assignment_loss‚0.5026957988537106ð´Dj_»Ù trainer/global_step‚4199 1loss/SRqqt1/detection_loss‚2.8094711179534593 1loss/SRqqt3/detection_loss‚1.5947117693722248 /loss/FBt2/detection_loss‚1.8880320649880629 _runtime‚3337.352381398 /loss/FRt1/assignment_loss‚5.067676197737455 2loss/SRqqt4/assignment_loss‚1.3590598597570702 0loss/FBt2/assignment_loss‚0.8217053161136462 .loss/FRt4/detection_loss‚2.301483518216345 /loss/FBt3/detection_loss‚0.9013022438855842 % _timestamp‚1.7884482218808846e+09 _step‚344 1loss/SRqqt1/assignment_loss‚2.559399309257666 2loss/SRqqt2/assignment_loss‚1.4877667814797284 /loss/FRt1/detection_loss‚2.8756379410624504 /loss/FRt3/detection_loss‚1.8763163623944767 /loss/FBt1/detection_loss‚2.5296903523531826 0loss/FRt3/assignment_loss‚1.8795100697931253 2loss/SRqqt3/assignment_loss‚1.3007558591517747 /loss/FRt2/detection_loss‚2.3160758874517806 /loss/FBt4/detection_loss‚3.3732869148254396 %loss/total_loss‚48.64978924612436 0loss/FBt3/assignment_loss‚0.2176822180299312 0loss/FBt4/assignment_loss‚0.5026957988537106 epoch‚3 /loss/FRt2/assignment_loss‚3.732748561717094 0loss/FRt4/assignment_loss‚3.1919942882325913 0loss/FBt1/assignment_loss‚0.6363084100355226 1loss/SRqqt2/detection_loss‚1.6932117567986857 0loss/SRqqt4/detection_loss‚1.733266646663348Øy£ “¼‚(:ˆ å›æԯʉý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.586' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.5172 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0rHi!K½‚(:À å›æÔÀÓâÿ* proc.memory.percent‚15.686726570129395 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd2080.out‚0 disk.rbd1.out‚ 17.6748046875 disk.dm-1.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125# proc.memory.rssMB‚ 2570.11328125 network.sent‚ 9.349489e+06 network.recv‚ 1.18959e+06$ disk.nvme0c0n1.out‚ 13099.6328125 disk.sda3.in‚0 disk.sda3.out‚ 127.5703125 cpu‚25.32810349697162 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0% disk./.usageGB‚300.36266326904297 disk.sda.out‚ 127.5703125 disk.sda1.in‚0& proc.memory.availableMB‚ 545.953125 proc.cpu.threads‚17" disk.nvme0n1.out‚ 13099.6328125$ memory_percent‚96.66776657104492* disk./.usagePercent‚20.157590353046267 disk.dm-1.out‚ 127.5703125 disk.drbd1716.out‚0 disk.sda.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.dm-0.out‚ 13099.6328125R“Ë¿“¾‚(:ˆ ô›æÔ–˜ý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.806' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.7612 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¨;<,Q¿‚(:Æ ô›æÔ”œ€ disk.drbd2265.in‚0 disk.drbd2080.out‚0( proc.memory.availableMB‚ 545.94140625 proc.cpu.threads‚17$ disk./.usageGB‚300.4130058288574 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0( disk.nvme0c0n1.out‚13148.11474609375 disk.sda1.in‚0* proc.memory.percent‚15.686726570129395 disk.sda.in‚0 network.recv‚ 1.19653e+06# disk.dm-0.out‚13148.11474609375& disk.nvme0n1.out‚13148.11474609375 disk.sda3.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 128.203125 disk.sda1.out‚0 disk.sda2.out‚0 cpu‚25.3269396337502 disk.drbd2265.out‚0 disk.rbd0.in‚0$ memory_percent‚96.66783809661865# proc.memory.rssMB‚ 2570.11328125 disk.rbd1.out‚ 17.70703125 disk.dm-1.out‚ 128.203125 disk.drbd1716.in‚0 disk.sda3.out‚ 128.203125 disk.sda2.in‚0 disk.dm-1.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 network.sent‚ 9.458298e+06) disk./.usagePercent‚20.16096888447564 disk.rbd0.out‚0ho¤2À- _wandbruntime‚3373  _runtime‚3373Ó°‰ Áà *â†îÀ_step‚345 ' lr-AdamW/pg1‚0.00010381315263359107 ' lr-AdamW/pg2‚0.00010381315263359107 trainer/global_step‚4249 % _timestamp‚1.7884482584884503e+09 _runtime‚3373.960079295IÌÆÎÂÈ _step‚345 ' lr-AdamW/pg1‚0.00010381315263359107 ' lr-AdamW/pg2‚0.00010381315263359107 trainer/global_step‚4249 % _timestamp‚1.7884482584884503e+09 _runtime‚3373.960079295Ùxöî2Ã- _wandbruntime‚3373  _runtime‚3373_XFÝ'Ä×O G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3553258851840 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚272922222898176 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚61617932 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35948956 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚250030787519488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚240028758811648 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚61617930 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3552602489344 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚250031461884928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚246478203033088 trainer/global_step‚4249 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3727251064320 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3540179696030 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 141712653 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚246478203033088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚269194804015616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚250030787519488 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚450 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚80094723 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3553258851840 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3727418882560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚346 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚246478185030144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884482584892044e+09 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚240028775851008 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61617930 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚250031461884928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56243626 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 141732693 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚80114761 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚269194780075520 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3552602489344 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20294220 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚243568284069922 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚80094723 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚3373.961113467 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚246478185030144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61617932 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚272922031139840 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3539525258274 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844£Ò²áâ'ÅÜO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚35948958 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚250030787519488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚240028758811648 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚61617930 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3552602489344 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚250031461884928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚246478203033088 trainer/global_step‚4249 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3727251064320 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3540179696030 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 141712653 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚246478203033088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚269194804015616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚250030787519488 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚450 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚80094723 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3553258851840 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3727418882560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _step‚346 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚246478185030144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884482584892044e+09 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚240028775851008 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚61617930 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚250031461884928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56243626 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 141732693 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚80114761 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚269194780075520 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3552602489344 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20294220 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚243568284069922 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚80094723 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚3373.961113467 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚246478185030144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚61617932 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚272922031139840 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3539525258274 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3553258851840 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚272922222898176 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚61617932 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚35948956 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 141765189 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚240083662942208 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚250087959986176 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3540341693238 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61631946 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3540996131202 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚243624004635446 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 141745123 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56255713 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3728271551488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3728101649408 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚250088634364928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚347 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35957126 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚80133241 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚246534538068992 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 141745123 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚80113177 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚269256430813184 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35957124 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3554078292992 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚ 3374.66286136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1µj"1ÈåM >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚35957124 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3554078292992 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚ 3374.66286136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 RE*í¬ŠºDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚272984726304768 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 141765189 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3553421917184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚246534556071936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚246534556071936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚243624676112770 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚61631946 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚250088634364928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 141765189 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚240083662942208 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚250087959986176 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3540341693238 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚61631946 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3540996131202 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚243624004635446 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 141745123 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56255713 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3728271551488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3728101649408 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚250088634364928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚347 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚35957126 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚80133241 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚246534538068992 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 141745123 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚80113177 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚269256430813184ÛU~ðõ2É- _wandbruntime‚3374  _runtime‚3374» ~LTÊÎ /loss/FRt4/detection_loss‚2.4312052823776424 0loss/SRqqt2/detection_loss‚1.626153339599741 1loss/SRqqt3/detection_loss‚1.3924835930105115 0loss/FRt3/assignment_loss‚1.8727923188454065 /loss/FBt1/assignment_loss‚0.440778986761608 .loss/SRqqt1/detection_loss‚2.4859333154228 % _timestamp‚1.7884482591929502e+09 _step‚348 .loss/FBt1/detection_loss‚3.118219160116636 .loss/FBt2/detection_loss‚3.464873343706131 /loss/FRt1/detection_loss‚3.2254141998291015 epoch‚3 1loss/FBt3/assignment_loss‚0.26647974592431495 0loss/SRqqt4/detection_loss‚1.806941716779362 /loss/FBt3/detection_loss‚0.8201472450507014 0loss/FRt4/assignment_loss‚2.7824826198954913 2loss/SRqqt2/assignment_loss‚1.3871824813942457 /loss/FRt2/detection_loss‚2.2169327600946964 trainer/global_step‚4249 /loss/FRt2/assignment_loss‚3.727237587829806 1loss/SRqqt1/assignment_loss‚2.586595108939542 2loss/SRqqt4/assignment_loss‚1.4315571964464404 &loss/total_loss‚51.210320897311206 _runtime‚3374.664506855 .loss/FRt1/assignment_loss‚5.66610249042511 0loss/FBt4/assignment_loss‚0.7207472388233457 .loss/FRt3/detection_loss‚2.064440997556234 /loss/FBt4/detection_loss‚3.6740494796207974 2loss/SRqqt3/assignment_loss‚1.3376123387305463 0loss/FBt2/assignment_loss‚0.6639583501309971ÏŸüYËÓ 1loss/SRqqt1/assignment_loss‚2.586595108939542 2loss/SRqqt4/assignment_loss‚1.4315571964464404 &loss/total_loss‚51.210320897311206 _runtime‚3374.664506855 .loss/FRt1/assignment_loss‚5.66610249042511 0loss/FBt4/assignment_loss‚0.7207472388233457 .loss/FRt3/detection_loss‚2.064440997556234 /loss/FBt4/detection_loss‚3.6740494796207974 2loss/SRqqt3/assignment_loss‚1.3376123387305463 0loss/FBt2/assignment_loss‚0.6639583501309971 /loss/FRt4/detection_loss‚2.4312052823776424 0loss/SRqqt2/detection_loss‚1.626153339599741 1loss/SRqqt3/detection_loss‚1.3924835930105115 0loss/FRt3/assignment_loss‚1.8727923188454065 /loss/FBt1/assignment_loss‚0.440778986761608 .loss/SRqqt1/detection_loss‚2.4859333154228 % _timestamp‚1.7884482591929502e+09 _step‚348 .loss/FBt1/detection_loss‚3.118219160116636 .loss/FBt2/detection_loss‚3.464873343706131 /loss/FRt1/detection_loss‚3.2254141998291015 epoch‚3 1loss/FBt3/assignment_loss‚0.26647974592431495 0loss/SRqqt4/detection_loss‚1.806941716779362 /loss/FBt3/detection_loss‚0.8201472450507014 0loss/FRt4/assignment_loss‚2.7824826198954913 2loss/SRqqt2/assignment_loss‚1.3871824813942457 /loss/FRt2/detection_loss‚2.2169327600946964 trainer/global_step‚4249 /loss/FRt2/assignment_loss‚3.727237587829806ܳ »À“Ì‚(:ˆ ƒœæÔËø–ý gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚180.132' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0264 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ç<ÓVÍ‚(:Ë ƒœæÔÅÓžÿ network.sent‚ 9.46355e+06 network.recv‚ 1.200048e+06 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚13175.66943359375 disk.sda3.out‚ 128.7578125* proc.memory.percent‚15.686726570129395 cpu‚25.32566091447586 disk.dm-1.out‚ 128.7578125# disk.dm-0.out‚13175.66943359375 disk.sda.out‚ 128.7578125 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.drbd1716.out‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125% disk./.usageGB‚300.28872299194336 disk.dm-1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.sda.in‚0 disk.sda1.in‚0( proc.memory.availableMB‚ 545.66796875) disk./.usagePercent‚20.15262815901009 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd1.out‚ 17.81640625& disk.nvme0n1.out‚13175.66943359375# proc.memory.rssMB‚ 2570.11328125 proc.cpu.threads‚17 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚96.669507026672365¸•¦“΂(:ˆ ’œæÔúÙâü gpu.0.gpu‚30.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚173.068' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.6136 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Iù}TÏ‚(:É ’œæÔÓ÷ãÿ disk.nvme0n1.in‚0.078125# memory_percent‚96.6681957244873' proc.memory.availableMB‚ 545.8828125* proc.memory.percent‚15.686726570129395 network.recv‚ 1.207186e+06 disk.sda.out‚ 129.3203125 disk.drbd1716.out‚0 network.sent‚ 9.572582e+06 disk.rbd0.out‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.drbd2080.out‚0 cpu‚25.32444343562782 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 disk.drbd2265.in‚0# proc.memory.rssMB‚ 2570.11328125) disk./.usagePercent‚20.16000245603388 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0# disk.dm-0.out‚13230.96337890625 disk.rbd1.out‚ 17.81640625 disk.dm-1.in‚0 disk.dm-1.out‚ 129.3203125 disk.drbd2266.out‚0 disk.sda.in‚0 disk.sda2.in‚0( disk.nvme0c0n1.out‚13230.96337890625 disk.sda1.in‚0& disk.nvme0n1.out‚13230.96337890625 disk.drbd2265.out‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚300.3986053466797 disk.sda3.out‚ 129.3203125 disk.drbd2266.in‚0 disk.rbd0.in‚0~O’Ђ(:‡ ¡œæÔ™º¹ÿ gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚174.955' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.991 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 %s>[Ñ‚(:Ð ¡œæÔŒ»ž disk.sda.in‚0 disk.dm-1.in‚0( disk.nvme0c0n1.out‚13275.08447265625 disk.dm-0.in‚0.078125 disk.sda.out‚ 129.94140625 disk.drbd2265.in‚0# disk.dm-0.out‚13275.08447265625# proc.memory.rssMB‚ 2570.11328125 network.sent‚ 9.577808e+06 disk.sda3.in‚0 disk.sda2.in‚0 proc.cpu.threads‚17& disk.nvme0n1.out‚13275.08447265625 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 129.94140625 disk.rbd0.in‚0 disk.sda1.in‚0( proc.memory.availableMB‚ 545.87890625 cpu‚25.323222026346237 network.recv‚ 1.210806e+06% disk./.usageGB‚300.42656326293945 disk.drbd1716.out‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.dm-1.out‚ 129.94140625 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.rbd1.in‚0* proc.memory.percent‚15.686726570129395 disk.sda1.out‚0 disk.rbd1.out‚ 17.87890625$ memory_percent‚96.66821956634521) disk./.usagePercent‚20.16187873525233 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.rbd0.out‚0‡@2Ò- _wandbruntime‚3410  _runtime‚3410~$WÉÓà ' lr-AdamW/pg1‚0.00010310550811218484 ' lr-AdamW/pg2‚0.00010310550811218484 trainer/global_step‚4299 % _timestamp‚1.7884482955047042e+09 _runtime‚3410.976298526 _step‚349 ¶¢ÎÔÈ trainer/global_step‚4299 % _timestamp‚1.7884482955047042e+09 _runtime‚3410.976298526 _step‚349 ' lr-AdamW/pg1‚0.00010310550811218484 ' lr-AdamW/pg2‚0.00010310550811218484Ý|Ü*2Õ- _wandbruntime‚3410  _runtime‚3410\ÈÝ'Ö×O F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 143339505 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚249295854975488 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚242773965339648 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚81020775 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚276047546177024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56877310 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚276047737935360 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚249295854975488 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36357356 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚249295836972544 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚242773982379008 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20519952 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3770397036544 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3593588295168 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚3410.977321326 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚252889425267712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36357358 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚81040813 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3580360597682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414ÞBtb×ÜO O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3594244657664 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚350 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚252890099633152 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚249295854975488 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36357356 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚249295836972544 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚242773982379008 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20519952 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3770397036544 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3593588295168 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚3410.977321326 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚252889425267712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36357358 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚81040813 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3580360597682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 143339505 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚249295854975488 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚242773965339648 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚81020775 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚276047546177024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚56877310 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚276047737935360 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚242828886509568 trainer/global_step‚4299 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62332746 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62332748 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 143371975 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 143392041 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3581831470610 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3581177032646 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 143371975 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3771079754240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 3411.6681519 _step‚351 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3594407723008 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56890015 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56889568 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚81059293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3771249656320 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36365524 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚81059293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚249352208014336 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36365526 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚249352190011392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚249352208014336 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 143392041 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3581831470610 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3581177032646 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 143371975 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3771079754240 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 3411.6681519 _step‚351 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3594407723008 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚56890015 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚56889568 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚81059293 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3771249656320 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36365524 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚81059293 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚249352208014336 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚36365526 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚249352190011392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚249352208014336 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚242828886509568 trainer/global_step‚4299 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚62332746 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚62332748 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0ßZTŸ32Û- _wandbruntime‚3411  _runtime‚34111 T [ÜÕ 1loss/SRqqt4/detection_loss‚1.5918055245327571 % _timestamp‚1.7884482961975732e+09 0loss/FRt2/assignment_loss‚3.0443709948483635 0loss/FBt1/assignment_loss‚0.3456145203487143 0loss/FBt2/assignment_loss‚0.5515918700493785 2loss/SRqqt3/assignment_loss‚1.2863299653810614 1loss/FBt3/assignment_loss‚0.21615274214285113 1loss/SRqqt2/detection_loss‚2.0132102133198218 .loss/FRt2/detection_loss‚2.199843184036367 2loss/SRqqt2/assignment_loss‚1.6322573895820163 /loss/FRt3/detection_loss‚2.0850732842316995 /loss/FBt2/detection_loss‚1.9291118135054905 /loss/FBt4/detection_loss‚3.2180990278720856 epoch‚3 /loss/FRt3/assignment_loss‚2.360418195907886 1loss/SRqqt1/detection_loss‚2.5926448504130044 1loss/SRqqt3/detection_loss‚1.1885717071811943 /loss/FBt3/detection_loss‚0.7259187429853073 %loss/total_loss‚49.08978256305397 /loss/FRt1/assignment_loss‚5.738644557339804 /loss/FRt4/assignment_loss‚2.867934760451317 2loss/SRqqt4/assignment_loss‚1.4434483031195307 0loss/FBt4/assignment_loss‚0.5698637366294861 /loss/FRt4/detection_loss‚2.4325648501515387 _step‚352 trainer/global_step‚4299 .loss/FRt1/detection_loss‚3.097363912633487 _runtime‚3411.669101522 2loss/SRqqt1/assignment_loss‚2.7717401865455837 /loss/FBt1/detection_loss‚3.1872082298452202&¤`ÝÚ /loss/FBt2/detection_loss‚1.9291118135054905 /loss/FBt4/detection_loss‚3.2180990278720856 epoch‚3 /loss/FRt3/assignment_loss‚2.360418195907886 1loss/SRqqt1/detection_loss‚2.5926448504130044 1loss/SRqqt3/detection_loss‚1.1885717071811943 /loss/FBt3/detection_loss‚0.7259187429853073 %loss/total_loss‚49.08978256305397 /loss/FRt1/assignment_loss‚5.738644557339804 /loss/FRt4/assignment_loss‚2.867934760451317 2loss/SRqqt4/assignment_loss‚1.4434483031195307 0loss/FBt4/assignment_loss‚0.5698637366294861 /loss/FRt4/detection_loss‚2.4325648501515387 _step‚352 trainer/global_step‚4299 .loss/FRt1/detection_loss‚3.097363912633487 _runtime‚3411.669101522 2loss/SRqqt1/assignment_loss‚2.7717401865455837 /loss/FBt1/detection_loss‚3.1872082298452202 1loss/SRqqt4/detection_loss‚1.5918055245327571 % _timestamp‚1.7884482961975732e+09 0loss/FRt2/assignment_loss‚3.0443709948483635 0loss/FBt1/assignment_loss‚0.3456145203487143 0loss/FBt2/assignment_loss‚0.5515918700493785 2loss/SRqqt3/assignment_loss‚1.2863299653810614 1loss/FBt3/assignment_loss‚0.21615274214285113 1loss/SRqqt2/detection_loss‚2.0132102133198218 .loss/FRt2/detection_loss‚2.199843184036367 2loss/SRqqt2/assignment_loss‚1.6322573895820163 /loss/FRt3/detection_loss‚2.0850732842316995à­9“Þ‚(:ˆ °œæÔÕä´ý gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.804' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.9608 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0%ëLß‚(:Á °œæÔ‘ë¹ÿ disk.drbd2266.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 130.3828125 network.recv‚ 1.215894e+06 disk.sda3.in‚0 disk.sda2.in‚0* proc.memory.percent‚15.686726570129395 cpu‚25.32206135427083 network.sent‚ 9.584351e+06 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚13324.94921875 disk.rbd0.in‚0 disk.drbd1716.in‚0# proc.memory.rssMB‚ 2570.11328125 disk.drbd2080.out‚0 disk.sda3.out‚ 130.3828125 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.dm-0.out‚13324.94921875 disk.nvme0n1.in‚0.078125# disk.nvme0n1.out‚13324.94921875 proc.cpu.threads‚17 disk.rbd0.out‚0 disk.sda.out‚ 130.3828125 disk.rbd1.out‚ 17.96484375 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.dm-1.in‚0( proc.memory.availableMB‚ 545.84765625 disk.sda.in‚0 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0# memory_percent‚96.6684103012085) disk./.usagePercent‚20.16293067021052$ disk./.usageGB‚300.4422378540039"f¦£‘à‚(:† ¿œæÔî…›ý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚185.48' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.096 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0EÈ1Pá‚(:Å ¿œæÔ溔ÿ* disk./.usagePercent‚20.165626429546073 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.sda3.out‚ 130.76953125 disk.dm-0.in‚0.078125$ memory_percent‚96.66845798492432 network.sent‚ 9.692106e+06 disk.rbd1.out‚ 18.13671875 disk.drbd2265.out‚0" disk.nvme0n1.out‚ 13370.3984375$ disk.nvme0c0n1.out‚ 13370.3984375 disk.sda.out‚ 130.76953125 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 545.83984375 network.recv‚ 1.221644e+06 disk.sda1.in‚0 disk.rbd1.in‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.dm-0.out‚ 13370.3984375* proc.memory.percent‚15.686726570129395 disk.sda1.out‚0 disk.dm-1.out‚ 130.76953125# proc.memory.rssMB‚ 2570.11328125 cpu‚25.320887687498082 disk.drbd2265.in‚0 disk.drbd2266.out‚0 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125% disk./.usageGB‚300.48240661621094 disk.sda.in‚0 disk.dm-1.in‚0 disk.drbd2080.in‚0¡R¥2â- _wandbruntime‚3448  _runtime‚3448;£!ZÉãà _step‚353 ' lr-AdamW/pg1‚0.00010238957711167079 ' lr-AdamW/pg2‚0.00010238957711167079 trainer/global_step‚4349 % _timestamp‚1.7884483325618327e+09 _runtime‚3448.033499563WÅéÎäÈ trainer/global_step‚4349 % _timestamp‚1.7884483325618327e+09 _runtime‚3448.033499563 _step‚353 ' lr-AdamW/pg1‚0.00010238957711167079 ' lr-AdamW/pg2‚0.00010238957711167079áZ“Ï2å- _wandbruntime‚3448  _runtime‚3448ð‚‡èi æ×O TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚81966865 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63019532 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3634574100992 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚275359853841920 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3621195937090 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚255748063015936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3635230463488 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20745463 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚275359877782016 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63019532 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _step‚354 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3635230463488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚279172972450816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚252113506917888 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57511664 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 JBDeviceStvÅtatsMonitor.on_train_batch_start/segment.large_pool.current‚430 trainer/global_step‚4349 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚245519188907008 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚81966865 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚255748063015936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3634574100992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3813118608896 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 % _timestamp‚1.7884483325625746e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚252113488914944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚81946827 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚249141039281854 _runtime‚3448.034506312 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚252113506917888 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚252113488914944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63019530 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 144986397 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36765758 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36765756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚81946827 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚255748737381376 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3621850374846 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756G¦}uâ'çÜO N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 144986397 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚36765758 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚36765756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚81946827 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚255748737381376 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3621850374846 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚81966865 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63019532 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3634574100992 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚275359853841920 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3621195937090 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚255748063015936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3635230463488 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20745463 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚275359877782016 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63019532 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _step‚354 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3635230463488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚279172972450816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚252113506917888 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚57511664 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 trainer/global_step‚4349 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚245519188907008 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚81966865 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚255748063015936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3634574100992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3813118608896 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 % _timestamp‚1.7884483325625746e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚252113488914944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚81946827 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚249141039281854 _runtime‚3448.034506312 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚252113506917888 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚252113488914944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63019530 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128âØgƒz2è- _wandbruntime‚3448  _runtime‚3448òŒVŸç&éáM =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3622012372054 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20749906 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚245574075998208 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚255805909861376 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚249196759847586 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3636049904640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚81985345 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3622666810018 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3635393528832 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚252169841953792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63033546 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚81985345 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 145018893 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚255805235482624 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36773924 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20750350 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚275421504579584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚81965281 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3814141783040 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚279235670302720 _runtime‚3448.711516405 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63033548 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚252169841953792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3635393528832 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57523830 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚252169859956736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63033546 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3813976075264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚252169859956736 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3636049904640 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚81965281 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 trainer/global_step‚4349 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚275421528519680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 145018893 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚57523830 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚252169859956736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚63033546 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3813976075264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚252169859956736 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3636049904640 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚81965281 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 trainer/global_step‚4349 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚275421528519680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 145018893 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3622012372054 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20749906 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚245574075998208 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚255805909861376 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚249196759847586 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3636049904640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚81985345 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3622666810018 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3635393528832 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚252169841953792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚63033546 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚81985345 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 145018893 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚255805235482624 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚36773924 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0ã¬02ë- _wandbruntime‚3448  _runtime‚3448ÙD0™WìÑ 1loss/FBt4/assignment_loss‚0.16633329589785717 1loss/SRqqt2/detection_loss‚2.1202251068923785 .loss/FBt4/detection_loss‚4.422432502110799 $ _timestamp‚1.788448333241086e+09 0loss/FBt1/assignment_loss‚0.3308930456660164 /loss/FRt1/assignment_loss‚4.988680198788643 2loss/SRqqt3/assignment_loss‚1.5946747522423232 2loss/SRqqt4/assignment_loss‚1.4559733608494634 .loss/FRt3/detection_loss‚2.274229564292486 .loss/FRt4/detection_loss‚2.288183226900281 /loss/FBt2/detection_loss‚2.0600774039824805 .loss/FRt2/detection_loss‚2.029484856724739 trainer/global_step‚4349 /loss/FBt3/detection_loss‚0.7348610782000708 %loss/total_loss‚50.33600805288024 _runtime‚3448.712599049 .loss/FRt2/assignment_loss‚4.00952034831047 2loss/SRqqt2/assignment_loss‚1.5909847497292187 .loss/FBt1/detection_loss‚3.899489070687975 epoch‚3 _step‚356 0loss/FBt2/assignment_loss‚0.7472008969634771 1loss/FBt3/assignment_loss‚0.22774535270813004 /loss/FRt1/detection_loss‚2.8851893469691277 0loss/SRqqt3/detection_loss‚1.260490516713564 0loss/FRt3/assignment_loss‚2.0523505529692008 1loss/SRqqt1/detection_loss‚2.7548403472735963 1loss/SRqqt4/detection_loss‚1.5222898067339607 0loss/FRt4/assignment_loss‚2.5431105472006887 2loss/SRqqt1/assignment_loss‚2.3767481240732917é,\íÖ .loss/FRt2/detection_loss‚2.029484856724739 trainer/global_step‚4349 /loss/FBt3/detection_loss‚0.7348610782000708 %loss/total_loss‚50.33600805288024 _runtime‚3448.712599049 .loss/FRt2/assignment_loss‚4.00952034831047 2loss/SRqqt2/assignment_loss‚1.5909847497292187 .loss/FBt1/detection_loss‚3.899489070687975 epoch‚3 _step‚356 0loss/FBt2/assignment_loss‚0.7472008969634771 1loss/FBt3/assignment_loss‚0.22774535270813004 /loss/FRt1/detection_loss‚2.8851893469691277 0loss/SRqqt3/detection_loss‚1.260490516713564 0loss/FRt3/assignment_loss‚2.0523505529692008 1loss/SRqqt1/detection_loss‚2.7548403472735963 1loss/SRqqt4/detection_loss‚1.5222898067339607 0loss/FRt4/assignment_loss‚2.5431105472006887 2loss/SRqqt1/assignment_loss‚2.3767481240732917 1loss/FBt4/assignment_loss‚0.16633329589785717 1loss/SRqqt2/detection_loss‚2.1202251068923785 .loss/FBt4/detection_loss‚4.422432502110799 $ _timestamp‚1.788448333241086e+09 0loss/FBt1/assignment_loss‚0.3308930456660164 /loss/FRt1/assignment_loss‚4.988680198788643 2loss/SRqqt3/assignment_loss‚1.5946747522423232 2loss/SRqqt4/assignment_loss‚1.4559733608494634 .loss/FRt3/detection_loss‚2.274229564292486 .loss/FRt4/detection_loss‚2.288183226900281 /loss/FBt2/detection_loss‚2.0600774039824805ääáî‚(:… ÎœæÔ÷ɬý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.05' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.21 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0_µ×~Xï‚(:Í ÎœæÔêÑ°ÿ disk.drbd2265.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 131.60546875 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0$ memory_percent‚96.66883945465088 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd1.out‚ 18.13671875 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 proc.cpu.threads‚17 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.sda3.out‚ 131.60546875 disk.drbd2266.in‚0# disk.dm-0.out‚13426.76708984375( proc.memory.availableMB‚ 545.77734375 network.recv‚ 1.226585e+06( disk.nvme0c0n1.out‚13426.76708984375 disk.rbd0.in‚0 disk.sda1.in‚0# proc.memory.rssMB‚ 2570.11328125 network.sent‚ 9.698813e+06 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda.in‚0* proc.memory.percent‚15.686726570129395 cpu‚25.31971644874144( disk./.usagePercent‚20.1683257729871$ disk./.usageGB‚300.5226287841797 disk.drbd2266.out‚0& disk.nvme0n1.out‚13426.76708984375 disk.dm-1.out‚ 131.60546875 disk.sda3.in‚0 disk.drbd1716.out‚0Zÿ£•“ð‚(:ˆ ÝœæÔš¤ý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.523' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.3046 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ícÝ“Zñ‚(:Ï ÝœæÔ„Ñ´ÿ* proc.memory.percent‚15.686726570129395 disk.sda2.out‚0 disk.rbd1.out‚ 18.19921875( disk.nvme0c0n1.out‚13483.20947265625 disk.rbd0.out‚0 disk.sda1.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0 disk.sda3.out‚ 132.36328125# disk.dm-0.out‚13483.20947265625 disk.rbd1.in‚0& disk.nvme0n1.out‚13483.20947265625 disk.dm-1.in‚0# memory_percent‚96.6688871383667( proc.memory.availableMB‚ 545.76953125# proc.memory.rssMB‚ 2570.11328125 network.recv‚ 1.231638e+06 disk.dm-0.in‚0.078125 proc.cpu.threads‚17 disk.rbd0.in‚0 network.sent‚ 9.806401e+06 disk.sda1.in‚0 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 132.36328125 disk.drbd1716.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 cpu‚25.318558154600712) disk./.usagePercent‚20.16941559706036% disk./.usageGB‚300.53886795043945 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda.in‚0 disk.sda.out‚ 132.36328125&·<œò‚(:‘ ìœæÔ¹Éþ gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚156.7' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚31.339999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Jy$¸Vó‚(:Ë ìœæÔÕ¾×€ disk.sda2.out‚0( disk.nvme0c0n1.out‚13543.58349609375( proc.memory.availableMB‚ 545.76953125 network.sent‚ 9.812804e+06 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.rbd1.out‚ 18.19921875# memory_percent‚96.6688871383667$ disk./.usageGB‚300.5560417175293 disk.rbd0.in‚0& disk.nvme0n1.out‚13543.58349609375 disk.dm-1.out‚ 132.9921875 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 132.9921875 disk.sda.in‚0 disk.sda1.in‚0 disk.dm-1.in‚0* proc.memory.percent‚15.686726570129395 disk.rbd0.out‚0# disk.dm-0.out‚13543.58349609375 disk.sda1.out‚0 proc.cpu.threads‚17 disk.dm-0.in‚0.078125 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125# proc.memory.rssMB‚ 2570.11328125 cpu‚25.317391195154357 network.recv‚ 1.23655e+06* disk./.usagePercent‚20.170568142979512 disk.drbd2266.out‚0 disk.sda.out‚ 132.9921875 disk.sda2.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚05SR2ô- _wandbruntime‚3484  _runtime‚3484ÜÄ“xÉõà ' lr-AdamW/pg1‚0.00010166550883099638 ' lr-AdamW/pg2‚0.00010166550883099638 trainer/global_step‚4399 % _timestamp‚1.7884483693907976e+09 _runtime‚3484.862397337 _step‚357ÁƒÎöÈ ' lr-AdamW/pg2‚0.00010166550883099638 trainer/global_step‚4399 % _timestamp‚1.7884483693907976e+09 _runtime‚3484.862397337 _step‚357 ' lr-AdamW/pg1‚0.00010166550883099638åA$2÷- _wandbruntime‚3484  _runtime‚3484¾gvÝ'ø×O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3662031276498 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20972453 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚251927081149262 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3676216269312 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63720332 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58146609 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚251926409672146 % _timestamp‚1.7884483693915422e+09 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3856222551552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚358 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 146593209 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3676216269312 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _runtime‚3484.863388317 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 146613249 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20972894 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚248264378395648 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63720330 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚254931140857344 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚82892917 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚254931140857344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚282298637216768 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚278442414665216 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3675559906816 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚82892917 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3675559906816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37174156 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63720332 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚254931158860288 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚258606700764160 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚258607375129600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58147052 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3662685714254 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚4399 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚258606700764160 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3856050539008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚248264395435008 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37174158 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚282298441264128 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚258607375129600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚82872879 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889=.¼FŽ%ùÜO _step‚358 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 146593209 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3676216269312 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _runtime‚3484.863388317 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 146613249 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚20972894 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚248264378395648 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚63720330 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚254931140857344 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚82892917 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚254931140857344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚282298637216768 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚278442414665216 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3675559906816 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚82892917 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3675559906816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37174156 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚63720332 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚254931158860288 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚258606700764160 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚258607375129600 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58147052 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3662685714254 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚4399 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚258606700764160 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3856050539008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚248264395435008 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37174158 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚282298441264128 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚258607375129600 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚82872879 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3662031276498 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚20972453 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚251927081149262 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3676216269312 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.ÊTLÑTcurrent‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚63720332 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58146609 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚251926409672146 % _timestamp‚1.7884483693915422e+09 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3856222551552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360æÛ£t2ú- _wandbruntime‚3485  _runtime‚3485µ±ÿ„ç&ûáM ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚359 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3676379334656 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚437 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚282361147175936 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 146625679 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚435 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 146645745 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20977414 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3857081773056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚251982801714994 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚248319299565568 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚82911397 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 146645745 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3677035710464 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3856926551040 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3663502149426 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58159303 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37182326 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63734348 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37182324 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚254987493896192 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3676379334656 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58159740 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚248319282526208 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20976979 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚82891333ŸX ì&üæM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3857081773056 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚251982801714994 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚248319299565568 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚82911397 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 146645745 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3677035710464 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3856926551040 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3663502149426 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58159303 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37182326 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚63734348 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37182324 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚254987493896192 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3676379334656 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58159740 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚248319282526208 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚20976979 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚82891333 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚359 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3676379334656 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚437 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚282361147175936 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 146625679 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚435 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 146645745 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚20977414 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844çèÉn2ý- _wandbruntime‚3485  _runtime‚3485ŽÙ£ßTþÎ 0loss/FRt3/assignment_loss‚2.0847499496058415 2loss/SRqqt3/assignment_loss‚1.1940691126663177 /loss/FRt4/detection_loss‚2.4805757128878643 0loss/SRqqt2/detection_loss‚1.908531036017076 1loss/SRqqt3/detection_loss‚1.2738538731927755 /loss/FBt4/assignment_loss‚0.591382191516459 /loss/FRt1/assignment_loss‚4.644550586568898 1loss/SRqqt1/assignment_loss‚2.031193163394928 1loss/SRqqt4/assignment_loss‚1.517482965285408 epoch‚3 0loss/FRt2/assignment_loss‚3.6976206709038126 /loss/FRt3/detection_loss‚1.8373996078837336 % _timestamp‚1.7884483700830686e+09 2loss/SRqqt2/assignment_loss‚1.5984012265250367 0loss/FBt1/assignment_loss‚0.7741945405062076 0loss/FBt2/assignment_loss‚0.5061138285702782 /loss/FRt1/detection_loss‚3.1842487310541085 1loss/SRqqt1/detection_loss‚2.6039266276359556 .loss/FBt2/detection_loss‚3.250196291340722 .loss/FRt2/detection_loss‚2.290981947021051 1loss/SRqqt4/detection_loss‚1.6072897191705375 %loss/total_loss‚47.82382547416263 /loss/FBt3/detection_loss‚0.7784843012117423 _step‚360 /loss/FRt4/assignment_loss‚1.718705316907481 0loss/FBt3/assignment_loss‚0.2143957063464248 -loss/FBt1/detection_loss‚2.80375092679804 .loss/FBt4/detection_loss‚3.231727441151937 trainer/global_step‚4399 _runtime‚3485.554554352‹¹ÆsYÿÓ 0loss/FBt2/assignment_loss‚0.5061138285702782 /loss/FRt1/detection_loss‚3.1842487310541085 1loss/SRqqt1/detection_loss‚2.6039266276359556 .loss/FBt2/detection_loss‚3.250196291340722 .loss/FRt2/detection_loss‚2.290981947021051 1loss/SRqqt4/detection_loss‚1.6072897191705375 %loss/total_loss‚47.82382547416263 /loss/FBt3/detection_loss‚0.7784843012117423 _step‚360 /loss/FRt4/assignment_loss‚1.718705316907481 0loss/FBt3/assignment_loss‚0.2143957063464248 -loss/FBt1/detection_loss‚2.80375092679804 .loss/FBt4/detection_loss‚3.231727441151937 trainer/global_step‚4399 _runtime‚3485.554554352 0loss/FRt3/assignment_loss‚2.0847499496058415 2loss/SRqqt3/assignment_loss‚1.1940691126663177 /loss/FRt4/detection_loss‚2.4805757128878643 0loss/SRqqt2/detection_loss‚1.908531036017076 1loss/SRqqt3/detection_loss‚1.2738538731927755 /loss/FBt4/assignment_loss‚0.591382191516459 /loss/FRt1/assignment_loss‚4.644550586568898 1loss/SRqqt1/assignment_loss‚2.031193163394928 1loss/SRqqt4/assignment_loss‚1.517482965285408 epoch‚3 0loss/FRt2/assignment_loss‚3.6976206709038126 /loss/FRt3/detection_loss‚1.8373996078837336 % _timestamp‚1.7884483700830686e+09 2loss/SRqqt2/assignment_loss‚1.5984012265250367 0loss/FBt1/assignment_loss‚0.7741945405062076è˜%¾“€‚(:ˆ ûœæÔæ‘‚ý gpu.0.gpu‚27.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚181.409' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0÷CÖiT‚(:É ûœæÔöŸùÿ disk.rbd0.out‚0 disk.sda1.out‚0 disk.rbd1.out‚ 18.3408203125& disk.nvme0n1.out‚13591.38427734375 disk.sda3.out‚ 133.421875 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 network.recv‚ 1.239834e+06 disk.dm-0.in‚0.078125# disk.dm-0.out‚13591.38427734375 disk.sda.in‚0 disk.dm-1.in‚0 disk.dm-1.out‚ 133.421875 cpu‚25.31626435661658 disk.sda3.in‚0 disk.drbd2266.out‚0 network.sent‚ 9.817971e+06$ disk./.usageGB‚300.5646629333496 disk.drbd2080.in‚0 disk.sda.out‚ 133.421875 disk.sda2.out‚0$ memory_percent‚96.67215347290039* disk./.usagePercent‚20.171146720006895 disk.drbd2265.out‚0 disk.drbd1716.out‚0& proc.memory.availableMB‚ 545.234375* proc.memory.percent‚15.686726570129395 proc.cpu.threads‚17 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚13591.38427734375 disk.drbd1716.in‚0# proc.memory.rssMB‚ 2570.11328125Ƥšž‚‚(:“ ŠæÔ±‰Åþ gpu.0.gpu‚48.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚133.889' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.777800000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Û:ôXƒ‚(:Í ŠæÔÑÃÅ€ disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.out‚ 133.98046875 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚13642.8564453125 disk.dm-0.in‚0.078125$ memory_percent‚96.67067527770996 network.recv‚ 1.24699e+06 disk.sda.out‚ 133.98046875 disk.drbd2266.in‚0 disk.rbd0.in‚0' proc.memory.availableMB‚ 545.4765625 proc.cpu.threads‚17 network.sent‚ 9.927155e+06 disk.rbd1.out‚ 18.3408203125 disk.sda.in‚0 cpu‚25.31512678910394% disk./.usageGB‚300.57032012939453 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚0* proc.memory.percent‚15.686726570129395 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 133.98046875% disk.nvme0n1.out‚13642.8564453125 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2266.out‚0" disk.dm-0.out‚13642.8564453125# proc.memory.rssMB‚ 2570.11328125* disk./.usagePercent‚20.171526379180175FµÊ2„- _wandbruntime‚3521  _runtime‚3521Z6„¯È… trainer/global_step‚4449 $ _timestamp‚1.788448406254825e+09 _runtime‚3521.726382391 _step‚361 ' lr-AdamW/pg1‚0.00010093345416490606 ' lr-AdamW/pg2‚0.00010093345416490606Éâáþꠂ ' lr-AdamW/pg1‚0.00010093345416490606 ' lr-AdamW/pg2‚0.00010093345416490606 trainer/global_step‚4449 $ _timestamp‚1.788448406254825e+09 _runtime‚3521.726382391 _step‚361é2ÂSU2‡- _wandbruntime‚3521  _runtime‚3521Öuíu‚ˆ×O KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚83798931 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64421130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚257748810802688 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚254713123016670 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚281524927608320 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚4449 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3716545712640 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚285424024605696 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚254712451539554 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 148220061 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚261466012877824 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58782183 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3899096997376 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚261465338512384 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚83818969 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _runtime‚3521.727253505 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚83798931 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3899262718464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64421132 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21200054 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚285424214266880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚281524951548416 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚83818969 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚261465338512384 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64421132 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58782612 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21199627 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3703521053662$ˆ"]â'‰ÜO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚64421132 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚58782612 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21199627 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3703521053662 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚83798931 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚64421130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚257748810802688 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚254713123016670 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚281524927608320 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚4449 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3716545712640 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚285424024605696 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚254712451539554 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 148220061 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚261466012877824 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚58782183 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3899096997376 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚261465338512384 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚83818969 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _runtime‚3521.727253505 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚83798931 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3899262718464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚64421132 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21200054 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚285424214266880 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚281524951548416 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚83818969 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚261465338512384 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872êÀ*¿2Š- _wandbruntime‚3522  _runtime‚3522ÈBÂíç&‹áM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚261522510979072 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21204546 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚285486724358144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚83837449 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3718021516288 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚254768843582402 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚64435146 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚257805145838592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21204120 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3899960558592 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3718021516288 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚251064506093568 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4449 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚83817385 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 148272597 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37590726 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚257805163841536 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 % _timestamp‚1.7884484069307795e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚64435146 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚251064489054208 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 _step‚363 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58794844 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚281586578345984 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚83837449 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3704337488834 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚426 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 148272597 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3717365140480 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37590724 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚261523185357824 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚257805163841536 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3717365140480 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58795272 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚83817385 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚281586602286080 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚3522.402758435 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 148252531 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3900122072064 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64435148 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚428 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3703683050870 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚285486538904576 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚254768172105078 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚58794844 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚281586578345984 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚83837449 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3704337488834 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚426 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 148272597 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3717365140480 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37590724 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚261523185357824 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚257805163841536 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3717365140480 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚58795272 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚83817385 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚281586602286080 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚3522.402758435 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 148252531 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3900122072064 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚64435148 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚428 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3703683050870 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚285486538904576 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚254768172105078 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚251064506093568 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4449 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚83817385 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 148272597 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚37590726ë;ëÕ2- _wandbruntime‚3522  _runtime‚3522¢ôS#ZŽÔ 0loss/FRt3/assignment_loss‚1.9808051311969757 2loss/SRqqt4/assignment_loss‚1.7238715725252405 1loss/FBt3/assignment_loss‚0.30019644907688164 1loss/FBt4/assignment_loss‚0.14198290245913844 .loss/FRt3/detection_loss‚1.938905720114708 .loss/FBt1/detection_loss‚2.355124415953954 trainer/global_step‚4449 0loss/FBt1/assignment_loss‚0.3667297941276047 /loss/FRt1/detection_loss‚3.0524118328094483 1loss/SRqqt4/detection_loss‚1.5852473871782422 /loss/FBt4/detection_loss‚3.0602563619613647 1loss/SRqqt1/assignment_loss‚2.543952938455802 /loss/FRt4/detection_loss‚2.5156886300375296 1loss/FBt2/assignment_loss‚0.22444393290269155 .loss/FRt2/detection_loss‚2.069244102520101 1loss/SRqqt2/detection_loss‚1.7539970315992832 epoch‚3 _step‚364 0loss/FRt2/assignment_loss‚3.3478382779102698 2loss/SRqqt2/assignment_loss‚1.5125480956797088 .loss/FBt2/detection_loss‚4.355619941438947 /loss/FRt1/assignment_loss‚4.164696559906006 /loss/FRt4/assignment_loss‚2.562999380189319 1loss/SRqqt3/detection_loss‚1.5330930991331115 1loss/SRqqt1/detection_loss‚2.8586359436695394 %loss/total_loss‚48.19238980775056 $ _timestamp‚1.788448406932252e+09 _runtime‚3522.404127658 2loss/SRqqt3/assignment_loss‚1.2627213850500993 /loss/FBt3/detection_loss‚0.9813789218545906Ý¿Þð_Ù 1loss/SRqqt1/assignment_loss‚2.543952938455802 /loss/FRt4/detection_loss‚2.5156886300375296 1loss/FBt2/assignment_loss‚0.22444393290269155 .loss/FRt2/detection_loss‚2.069244102520101 1loss/SRqqt2/detection_loss‚1.7539970315992832 epoch‚3 _step‚364 0loss/FRt2/assignment_loss‚3.3478382779102698 2loss/SRqqt2/assignment_loss‚1.5125480956797088 .loss/FBt2/detection_loss‚4.355619941438947 /loss/FRt1/assignment_loss‚4.164696559906006 /loss/FRt4/assignment_loss‚2.562999380189319 1loss/SRqqt3/detection_loss‚1.5330930991331115 1loss/SRqqt1/detection_loss‚2.8586359436695394 %loss/total_loss‚48.19238980775056 $ _timestamp‚1.788448406932252e+09 _runtime‚3522.404127658 2loss/SRqqt3/assignment_loss‚1.2627213850500993 /loss/FBt3/detection_loss‚0.9813789218545906 0loss/FRt3/assignment_loss‚1.9808051311969757 2loss/SRqqt4/assignment_loss‚1.7238715725252405 1loss/FBt3/assignment_loss‚0.30019644907688164 1loss/FBt4/assignment_loss‚0.14198290245913844 .loss/FRt3/detection_loss‚1.938905720114708 .loss/FBt1/detection_loss‚2.355124415953954 trainer/global_step‚4449 0loss/FBt1/assignment_loss‚0.3667297941276047 /loss/FRt1/detection_loss‚3.0524118328094483 1loss/SRqqt4/detection_loss‚1.5852473871782422 /loss/FBt4/detection_loss‚3.0602563619613647ì]íÔö“‚(:ˆ ™æÔ¡û…þ gpu.0.gpu‚51.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.627' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9254 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0E4¼òU‘‚(:Ê ™æÔ¬ìòþ disk.drbd2265.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 134.4453125' proc.memory.availableMB‚ 545.9140625# proc.memory.rssMB‚ 2570.11328125* proc.memory.percent‚15.686726570129395 proc.cpu.threads‚17 network.sent‚ 9.931424e+06* disk./.usagePercent‚20.171921398805505 cpu‚25.31403218073392 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚13688.3525390625 disk.sda3.in‚0 disk.sda1.out‚0$ memory_percent‚96.66800498962402 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 18.4033203125$ disk./.usageGB‚300.5762062072754 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0" disk.dm-0.out‚13688.3525390625 disk.sda1.in‚0 disk.dm-1.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚13688.3525390625 disk.drbd2266.in‚0 network.recv‚ 1.249404e+06 disk.sda3.out‚ 134.4453125 disk.sda2.in‚0 disk.dm-1.out‚ 134.4453125 disk.rbd0.out‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0°8û™‘’‚(:† ¨æÔÜžïü gpu.0.gpu‚30.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.94' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.388 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡5€MZ“‚(:Ï ¨æÔГêÿ disk.sda3.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.rbd1.out‚ 18.4033203125* proc.memory.percent‚15.686726570129395 disk.drbd1716.out‚0 disk.sda3.out‚ 135.0703125 disk.drbd2266.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 network.sent‚ 1.0040419e+07* disk./.usagePercent‚20.171007195900735 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.dm-1.in‚0 proc.cpu.threads‚17 network.recv‚ 1.256079e+06# disk.dm-0.out‚13744.72705078125 disk.sda.out‚ 135.0703125 disk.drbd1716.in‚0( disk.nvme0c0n1.out‚13744.72705078125' proc.memory.availableMB‚ 545.6640625# proc.memory.rssMB‚ 2570.11328125 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 135.0703125 disk.drbd2080.in‚0$ memory_percent‚96.66953086853027% disk./.usageGB‚300.56258392333984 disk.sda.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.sda1.out‚0 disk.rbd1.in‚0& disk.nvme0n1.out‚13744.72705078125 disk.drbd2080.out‚0 cpu‚25.31283874366572ᙆO“”‚(:ˆ ·æÔé•ñü gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.261' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8522 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚01w—§]•‚(:Ò ·æÔž‹ñÿ disk.sda1.in‚0& disk.nvme0n1.out‚13790.10595703125 network.sent‚ 1.0045651e+07$ disk./.usageGB‚300.6068534851074( disk.nvme0c0n1.out‚13790.10595703125 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚13790.10595703125 disk.sda2.out‚0( proc.memory.availableMB‚ 545.66015625 disk.rbd0.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 18.544921875 disk.dm-1.out‚ 135.49609375$ memory_percent‚96.66955471038818 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.drbd2265.in‚0# proc.memory.rssMB‚ 2570.11328125 cpu‚25.311668373928104 proc.cpu.threads‚17 network.recv‚ 1.259357e+06 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0* disk./.usagePercent‚20.173978163335594 disk.sda1.out‚0 disk.dm-1.in‚0 disk.drbd1716.in‚0* proc.memory.percent‚15.686726570129395 disk.rbd0.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.sda.out‚ 135.49609375 disk.sda3.out‚ 135.49609375“trå2–- _wandbruntime‚3558  _runtime‚3558KûKLÉ—Ã trainer/global_step‚4499 % _timestamp‚1.7884484425458863e+09 _runtime‚3558.017464358 _step‚365 ' lr-AdamW/pg1‚0.00010019356567249498 ' lr-AdamW/pg2‚0.00010019356567249498åšTÎ˜È trainer/global_step‚4499 % _timestamp‚1.7884484425458863e+09 _runtime‚3558.017464358 _step‚365 ' lr-AdamW/pg1‚0.00010019356567249498 ' lr-AdamW/pg2‚0.00010019356567249498ívØÿ^2™- _wandbruntime‚3558  _runtime‚3558‡n·ZÜ'šÖO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚257499164884078 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚257498493406962 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚448 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚264323976260608 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21425220 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚288549721772544 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 149866953 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 149846913 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59416624 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚260566444742144 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37990956 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21425666 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59416176 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚260566462745088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65121930 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65121932 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚84745021 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 _runtime‚ 3558.01834985 _step‚366 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3757531518464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚264323976260608 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚260566462745088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3757531518464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3744356393070 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65121930 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37990958 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚288549527917056 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚284607464491520 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3743701955314 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3942063425536 % _timestamp‚1.7884484425465689e+09: ùyá'›ÛO E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 _runtime‚ 3558.01834985 _step‚366 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3757531518464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚264323976260608 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚260566462745088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3757531518464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3744356393070 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚65121930 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚37990958 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚288549527917056 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚284607464491520 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3743701955314 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3942063425536 % _timestamp‚1.7884484425465689e+09 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚257499164884078 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚257498493406962 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚448 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚264323976260608 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21425220 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚288549721772544 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 149866953 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 149846913 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚59416624 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚260566444742144 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚37990956 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21425666 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚59416176 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚260566462745088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚65121930 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65121932 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚84745021îêAT2œ- _wandbruntime‚3558  _runtime‚3558ä=É… àM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚260622797780992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚264381148727296 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37999124 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚288612036577792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚284669115229184 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65135948 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.alûº¢žalocated‚264381823106048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3758350946304 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚4499 _runtime‚ 3558.70814672 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚84743437 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _step‚367 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚257554885449810 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65135948 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59429262 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚84763501 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚253809695582208 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3744518390278 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3759007322112 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 % _timestamp‚1.7884484432359188e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚84763501 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚260622815783936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 149879383 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 149879383 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65135946 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59428804 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21429680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3943087056384 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21430136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚260622815783936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3745172828242 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 149899449 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430݆޿ë&žåM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚21430136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚260622815783936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3745172828242 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 149899449 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚260622797780992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚264381148727296 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚37999124 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚288612036577792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚284669115229184 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚65135948 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚264381823106048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3758350946304 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚4499 _runtime‚ 3558.70814672 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚84743437 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _step‚367 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚257554885449810 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65135948 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚59429262 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚84763501 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚253809695582208 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3744518390278 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3759007322112 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 % _timestamp‚1.7884484432359188e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚84763501 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚260622815783936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 149879383 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 149879383 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65135946 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚59428804 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21429680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3943087056384 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066ïž6ÍË2Ÿ- _wandbruntime‚3558  _runtime‚3558®û€R Ì /loss/FRt1/assignment_loss‚4.795278766575982 2loss/SRqqt4/assignment_loss‚1.7049887040777811 1loss/SRqqt3/detection_loss‚1.2452523767475099 0loss/FRt2/assignment_loss‚3.4474527433514597 2loss/SRqqt2/assignment_loss‚1.9249022199047936 /loss/FBt3/detection_loss‚0.7653634294306171 %loss/total_loss‚48.52572921571651 trainer/global_step‚4499 /loss/FRt4/assignment_loss‚3.433706140766541 epoch‚3 0loss/SRqqt1/detection_loss‚2.823898060247302 0loss/SRqqt2/detection_loss‚2.249439483218723 1loss/SRqqt4/detection_loss‚1.7915602281453118 0loss/FBt1/assignment_loss‚0.7892509400189283 0loss/FBt2/assignment_loss‚0.4532379170918527 /loss/FBt4/detection_loss‚2.4544260799884796 _runtime‚3558.709490085 _step‚368 2loss/SRqqt1/assignment_loss‚2.4456382235512137 0loss/FBt3/assignment_loss‚0.2258295890581633 /loss/FBt1/detection_loss‚2.8128752797842025 $ _timestamp‚1.788448443237993e+09 -loss/FRt2/detection_loss‚2.16251554787159 .loss/FRt3/assignment_loss‚1.97392226934433 2loss/SRqqt3/assignment_loss‚0.9725514396670318 0loss/FBt4/assignment_loss‚0.5926164537668228 .loss/FRt1/detection_loss‚2.815709447159487 -loss/FRt3/detection_loss‚2.03069361269474 .loss/FRt4/detection_loss‚2.468085275962949 /loss/FBt2/detection_loss‚2.1465349872907002%îúHW¡Ñ 0loss/SRqqt1/detection_loss‚2.823898060247302 0loss/SRqqt2/detection_loss‚2.249439483218723 1loss/SRqqt4/detection_loss‚1.7915602281453118 0loss/FBt1/assignment_loss‚0.7892509400189283 0loss/FBt2/assignment_loss‚0.4532379170918527 /loss/FBt4/detection_loss‚2.4544260799884796 _runtime‚3558.709490085 _step‚368 2loss/SRqqt1/assignment_loss‚2.4456382235512137 0loss/FBt3/assignment_loss‚0.2258295890581633 /loss/FBt1/detection_loss‚2.8128752797842025 $ _timestamp‚1.788448443237993e+09 -loss/FRt2/detection_loss‚2.16251554787159 .loss/FRt3/assignment_loss‚1.97392226934433 2loss/SRqqt3/assignment_loss‚0.9725514396670318 0loss/FBt4/assignment_loss‚0.5926164537668228 .loss/FRt1/detection_loss‚2.815709447159487 -loss/FRt3/detection_loss‚2.03069361269474 .loss/FRt4/detection_loss‚2.468085275962949 /loss/FBt2/detection_loss‚2.1465349872907002 /loss/FRt1/assignment_loss‚4.795278766575982 2loss/SRqqt4/assignment_loss‚1.7049887040777811 1loss/SRqqt3/detection_loss‚1.2452523767475099 0loss/FRt2/assignment_loss‚3.4474527433514597 2loss/SRqqt2/assignment_loss‚1.9249022199047936 /loss/FBt3/detection_loss‚0.7653634294306171 %loss/total_loss‚48.52572921571651 trainer/global_step‚4499 /loss/FRt4/assignment_loss‚3.433706140766541 epoch‚3ð°_òŸ“¢‚(:ˆ ÆæÔüãŠý gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚140.583' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚28.1166 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0—!7°E£‚(:º ÆæÔÃÔÆÿ network.recv‚ 1.26487e+06% disk.nvme0c0n1.out‚13843.30859375 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.sda3.out‚ 136.09375 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 136.09375$ memory_percent‚96.66976928710938 network.sent‚ 1.0052518e+07* proc.memory.percent‚15.686726570129395 cpu‚25.310582508451002 disk.drbd2265.out‚0 disk.rbd0.in‚0 disk.dm-0.out‚13843.30859375 disk.sda.out‚ 136.09375 disk.drbd2265.in‚0 disk.drbd2266.in‚0 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.sda2.out‚0 disk.drbd1716.in‚0# proc.memory.availableMB‚545.625# proc.memory.rssMB‚ 2570.11328125* disk./.usagePercent‚20.174867533509545$ disk./.usageGB‚300.6201057434082 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.sda3.in‚0 disk.rbd1.out‚ 18.544921875# disk.nvme0n1.out‚13843.30859375 disk.dm-1.in‚0­S“¤‚(:ˆ ÕæÔñ”ý gpu.0.gpu‚41.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚131.369' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.2738 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0”]jL¥‚(:Á ÕæÔæ¨Ûÿ disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0( proc.memory.availableMB‚ 545.25390625* proc.memory.percent‚15.686726570129395 cpu‚25.309498668365812 disk.dm-1.out‚ 137.0078125# proc.memory.rssMB‚ 2570.11328125) disk./.usagePercent‚20.17578813660267$ disk./.usageGB‚300.6338233947754 disk.sda1.in‚0 disk.rbd1.out‚ 18.607421875$ memory_percent‚96.67203426361084 network.sent‚ 1.0163423e+07 network.recv‚ 1.27685e+06 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.dm-0.out‚ 13875.3984375 disk.sda2.in‚0 disk.sda2.out‚0" disk.nvme0n1.out‚ 13875.3984375 disk.dm-1.in‚0 disk.sda3.in‚0 disk.sda3.out‚ 137.0078125 disk.rbd0.out‚0 disk.sda.in‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0$ disk.nvme0c0n1.out‚ 13875.3984375 proc.cpu.threads‚17 disk.dm-0.in‚0.078125 disk.sda.out‚ 137.0078125 disk.sda1.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0„.Ä2¦- _wandbruntime‚3595  _runtime‚3595i˜…ǧÁ & lr-AdamW/pg1‚9.944599754541599e-05 & lr-AdamW/pg2‚9.944599754541599e-05 trainer/global_step‚4549 % _timestamp‚1.7884484795438206e+09 _runtime‚3595.015444114 _step‚369ÂÜð‹Ì¨Æ _runtime‚3595.015444114 _step‚369 & lr-AdamW/pg1‚9.944599754541599e-05 & lr-AdamW/pg2‚9.944599754541599e-05 trainer/global_step‚4549 % _timestamp‚1.7884484795438206e+09ña‚2©- _wandbruntime‚3595  _runtime‚3595±¢å·É"ª×O TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚263384096684544 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60050256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚291675112446976 % _timestamp‚1.7884484795445356e+09 _step‚370 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚267182614008832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 151473765 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚260285206751486 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚267182614008832 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3784537294722 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚4549 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚263384096684544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚263384114687488 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65822732 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚85671073 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38399358 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3798517324288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚291674931174400 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚85651035 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60049810 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38399356 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21650454 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚85671073 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚3595.016330226 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚287690025314816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚267183288374272 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚263384114687488 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚444 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allKD ocated‚267183288374272 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3798517324288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3985087132160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚256499997979648 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3984929799680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3799173686784 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚260284535274370Í©{÷â'«ÜO SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚267182614008832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 151473765 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚260285206751486 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚267182614008832 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3784537294722 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚4549 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚263384096684544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚263384114687488 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚65822732 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚85671073 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38399358 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3798517324288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚291674931174400 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚85651035 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚60049810 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38399356 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21650454 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚85671073 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚3595.016330226 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚287690025314816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚267183288374272 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚263384114687488 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚444 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚267183288374272 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3798517324288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3985087132160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚256499997979648 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3984929799680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3799173686784 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚260284535274370 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚263384096684544 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60050256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚291675112446976 % _timestamp‚1.7884484795445356e+09 _step‚370 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038òý» 2¬- _wandbruntime‚3595  _runtime‚3595U““Õæ&­àM REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65836748 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3799336752128 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚263440467726336 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3799993127936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3985944218624 $ _timestamp‚1.788448480243324e+09 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60062416 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 151526301 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚260340255839894 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60062865 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _step‚371 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚256554902110208 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚3595.715479566 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3985772219392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚267240460854272 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65836746 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 151506235 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚263440449723392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚85669489 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38407526 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚267239786475520 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚287751676052480 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 151526301 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3786008167650 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚85689553 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0¦oI‰ë&®åM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚263440449723392 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60062416 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 151526301 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚260340255839894 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60062865 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _step‚371 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚256554902110208 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚3595.715479566 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3985772219392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚267240460854272 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚65836746 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 151506235 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚263440449723392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚85669489 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38407526 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚267239786475520 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚287751676052480 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 151526301 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3786008167650 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚85689553 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚65836748 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3799336752128 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚263440467726336 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3799993127936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3985944218624 $ _timestamp‚1.788448480243324e+09 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚270041251757056 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚294800417804288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚374 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60684254 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66523532 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4027879546368 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 153100617 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21876051 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚86577087 _runtime‚3632.086932619 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚86577087 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4028041073152 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚259245221547008 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3839503130112 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚86597125 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚447 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 # _timestamp‚1.78844851661505e+09 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3826027071886 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚266201766629888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38807758 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚270041926122496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚266201748626944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚263070577141778 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66523530 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚266201748626944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚294800603271168 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3840159492608 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3839503130112 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 153120657 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚445 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38807756 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21876496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚66523532 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚263071248618894 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚259245204507648 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚270041251757056 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚294800417804288 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚374 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚60684254 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚66523532 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4027879546368 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 153100617 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚21876051 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚86577087 _runtime‚3632.086932619 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚86577087 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4028041073152 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚259245221547008 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3839503130112 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚86597125 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚447 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 # _timestamp‚1.78844851661505e+09 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3826027071886 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚266201766629888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚38807758 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚270041926122496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚266201748626944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚263070577141778 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚66523530 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚266201748626944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚294800603271168 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3840159492608 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3839503130112 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 153120657 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚445 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚38807756 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚21876496öO_‚2¾- _wandbruntime‚3632  _runtime‚36323–ÀÕà¿áM D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3840978933760 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚263126969184626 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚442 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 153133087 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚259300125677568 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚290834212935680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 153133087 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚66537546 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚86595541 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 153153153 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3826843507058 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚266258119668736 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21880579 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚375 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 153153153 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚86615605 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60696503 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚294862927022592 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚3632.785508287 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3840322557952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freedÁ€Íâ‚270098424223744 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38815926 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3826189069094 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66537546 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4028901637632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 % _timestamp‚1.7884485173134577e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3840322557952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚290834188995584 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4028738027008 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66537548 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38815924 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚294863114573312 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚263126297707302 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚270099098602496 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66537548 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚266258101665792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60696945 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚266258101665792 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚86615605 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚86595541 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 153153153 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3826843507058 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚266258119668736 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚21880579 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚375 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 153153153 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚86615605 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚60696503 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚294862927022592 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚3632.785508287 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3840322557952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚270098424223744 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚38815926 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3826189069094 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚66537546 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4028901637632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 % _timestamp‚1.7884485173134577e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3840322557952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚290834188995584 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4028738027008 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚66537548 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚38815924 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚294863114573312 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚263126297707302 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚270099098602496 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚66537548 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚266258101665792 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚60696945 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚266258101665792 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚86615605 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚86595541 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3840978933760 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚263126969184626 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚442 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 153133087 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚259300125677568 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚290834212935680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 153133087 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844÷[êRD2Á- _wandbruntime‚3632  _runtime‚36323WÂÑ 2loss/SRqqt2/assignment_loss‚1.8442190291871334 2loss/SRqqt4/assignment_loss‚1.5377571963920043 0loss/FBt3/assignment_loss‚0.1740277562547871 0loss/SRqqt1/detection_loss‚2.475308394059539 /loss/FRt1/assignment_loss‚4.246793259745059 0loss/FRt4/assignment_loss‚3.2944316675788476 0loss/FBt1/assignment_loss‚0.4673927814274066 /loss/FRt3/detection_loss‚1.8381351252645255 .loss/FRt4/detection_loss‚2.971043611043378 .loss/FBt4/detection_loss‚3.330255448818207 %loss/total_loss‚49.34787873607838 .loss/FRt2/detection_loss‚2.148949198266293 /loss/FRt2/assignment_loss‚3.488085372650877 0loss/FRt3/assignment_loss‚1.5302978706359864 2loss/SRqqt3/assignment_loss‚1.3041975234805698 0loss/FBt2/assignment_loss‚0.3197227207997912 1loss/FBt4/assignment_loss‚0.48282888531684875 1loss/SRqqt2/detection_loss‚1.8999800647006315 % _timestamp‚1.7884485173150527e+09 0loss/SRqqt3/detection_loss‚1.180364854214057 /loss/FBt2/detection_loss‚3.3500958740711213 1loss/SRqqt4/detection_loss‚2.0528047892909784 _step‚376 /loss/FRt1/detection_loss‚3.3818131296531013 .loss/FBt1/detection_loss‚2.723200097680092 /loss/FBt3/detection_loss‚0.9081211323634937 1loss/SRqqt1/assignment_loss‚2.398052953183651 epoch‚3 trainer/global_step‚4599 _runtime‚3632.786556556‹‡ Ø\ÃÖ .loss/FRt2/detection_loss‚2.148949198266293 /loss/FRt2/assignment_loss‚3.488085372650877 0loss/FRt3/assignment_loss‚1.5302978706359864 2loss/SRqqt3/assignment_loss‚1.3041975234805698 0loss/FBt2/assignment_loss‚0.3197227207997912 1loss/FBt4/assignment_loss‚0.48282888531684875 1loss/SRqqt2/detection_loss‚1.8999800647006315 % _timestamp‚1.7884485173150527e+09 0loss/SRqqt3/detection_loss‚1.180364854214057 /loss/FBt2/detection_loss‚3.3500958740711213 1loss/SRqqt4/detection_loss‚2.0528047892909784 _step‚376 /loss/FRt1/detection_loss‚3.3818131296531013 .loss/FBt1/detection_loss‚2.723200097680092 /loss/FBt3/detection_loss‚0.9081211323634937 1loss/SRqqt1/assignment_loss‚2.398052953183651 epoch‚3 trainer/global_step‚4599 _runtime‚3632.786556556 2loss/SRqqt2/assignment_loss‚1.8442190291871334 2loss/SRqqt4/assignment_loss‚1.5377571963920043 0loss/FBt3/assignment_loss‚0.1740277562547871 0loss/SRqqt1/detection_loss‚2.475308394059539 /loss/FRt1/assignment_loss‚4.246793259745059 0loss/FRt4/assignment_loss‚3.2944316675788476 0loss/FBt1/assignment_loss‚0.4673927814274066 /loss/FRt3/detection_loss‚1.8381351252645255 .loss/FRt4/detection_loss‚2.971043611043378 .loss/FBt4/detection_loss‚3.330255448818207 %loss/total_loss‚49.34787873607838ø>/ø~“Ä‚(:ˆ ‘žæԯƬý gpu.0.gpu‚42.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚175.407' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.0814 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0=ûÉ4YÅ‚(:Î ‘žæÔÝ·¨€% disk./.usageGB‚300.69802474975586' disk.nvme0c0n1.out‚14095.8310546875 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 139.48828125 disk.sda2.out‚0# memory_percent‚96.6728687286377# proc.memory.rssMB‚ 2570.11328125 disk.sda3.out‚ 139.48828125 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0' proc.memory.availableMB‚ 545.1171875 proc.cpu.threads‚17 network.recv‚ 1.302225e+06 cpu‚25.305048460462704 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0% disk.nvme0n1.out‚14095.8310546875 disk.drbd2265.in‚0 disk.rbd1.out‚ 19.099609375 network.sent‚ 1.0293179e+07 disk.sda3.in‚0 disk.sda.in‚0* disk./.usagePercent‚20.180096743403936 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125" disk.dm-0.out‚14095.8310546875 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0* proc.memory.percent‚15.686726570129395 disk.sda1.out‚0 disk.dm-1.in‚0 disk.dm-1.out‚ 139.48828125 disk.drbd1716.out‚0Si=žÆ‚(:“  žæÔØ“Œý gpu.0.gpu‚56.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚134.865' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.973000000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Û÷¿áYÇ‚(:Î  žæÔª¾ÿ disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2266.out‚0 disk.sda.out‚ 140.11328125 disk.sda1.out‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125) disk./.usagePercent‚20.18063563926351! disk./.usageGB‚300.7060546875 disk.sda3.out‚ 140.11328125 disk.rbd0.out‚0 disk.rbd1.in‚0$ memory_percent‚96.67141437530518( proc.memory.availableMB‚ 545.35546875# proc.memory.rssMB‚ 2570.11328125 cpu‚25.304028664641947 proc.cpu.threads‚17 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.sda2.in‚0( disk.nvme0c0n1.out‚14157.55224609375# disk.dm-0.out‚14157.55224609375 disk.sda1.in‚0& disk.nvme0n1.out‚14157.55224609375 disk.sda3.in‚0 disk.rbd0.in‚0 disk.drbd2080.out‚0 network.recv‚ 1.309237e+06 disk.dm-1.in‚0 disk.dm-1.out‚ 140.11328125* proc.memory.percent‚15.686726570129395 network.sent‚ 1.0401875e+07 disk.sda.in‚0 disk.rbd1.out‚ 19.099609375 disk.drbd2080.in‚0’y-2È- _wandbruntime‚3668  _runtime‚3668§DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67224332 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39216156 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚272900563870720 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3881145298432 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3880488935936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67224332 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚269019400569344 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39216158 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚297925962301440 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7884485532593918e+09 _runtime‚3668.731158129 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚293855075141120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61319454 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚261990428075008 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4070701693440 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚265857290486302 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22102878 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3881145298432 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚420 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚272899889505280 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚269019418572288 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚269019418572288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944ÀÙÞ¥ÍÜO ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3880488935936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67224332 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚269019400569344 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39216158 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚297925962301440 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7884485532593918e+09 _runtime‚3668.731158129 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚293855075141120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚61319454 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚261990428075008 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4070701693440 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚265857290486302 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22102878 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3881145298432 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚420 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚272899889505280 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚269019418572288 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚269019418572288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67224332 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39216156 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚272900563870720 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3881145298432 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952ú.w+ª2Î- _wandbruntime‚3669  _runtime‚3669#’¡ç&ÏáM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67238346 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67238346 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚297988475746304 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚293916749818880 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67238348 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚87521593 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4071566511104 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61331732 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚4649 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚87521593 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚434 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39224324 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚269075771611136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3881308363776 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚272957061971968 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39224326 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚269075771611136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3881964739584 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚272957736350720 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22107842 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4071725927424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3867678846466 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚269075753608192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3881964739584 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚262045332205568 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚262045315166208 % _timestamp‚1.7884485539494226e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 154780005 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚87541657 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 _step‚379 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚272957736350720 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430zºì&ÐæM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚269075753608192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3881964739584 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚262045332205568 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚262045315166208 % _timestamp‚1.7884485539494226e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 154780005 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚87541657 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 _step‚379 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚272957736350720 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚67238346 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚67238346 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚297988475746304 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚293916749818880 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67238348 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚87521593 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4071566511104 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61331732 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚4649 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚87521593 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚434 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39224324 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚269075771611136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3881308363776 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚272957061971968 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39224326 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚269075771611136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3881964739584 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚272957736350720 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22107842 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4071725927424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3867678846466 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0û¶‰2Ñ- _wandbruntime‚3669  _runtime‚3669ŒŠåTÒÎ /loss/FRt2/detection_loss‚2.1862923758370534 /loss/FRt3/detection_loss‚1.9342655090858107 0loss/SRqqt4/detection_loss‚1.598136268556118 _step‚380 2loss/SRqqt1/assignment_loss‚2.0383855141699314 2loss/SRqqt4/assignment_loss‚1.6526715335746607 &loss/total_loss‚50.533189647397045 epoch‚3 0loss/FRt4/assignment_loss‚3.5211879771489363 1loss/SRqqt3/assignment_loss‚1.281926613417454 1loss/SRqqt3/detection_loss‚1.3986825530300848 .loss/FBt1/detection_loss‚2.939144325256348 .loss/FBt2/assignment_loss‚0.46532531181374 .loss/FRt1/detection_loss‚3.176379151344299 /loss/FBt2/detection_loss‚2.2255507707595825 % _timestamp‚1.7884485539515333e+09 /loss/FBt1/assignment_loss‚0.854157385347935 0loss/FBt4/assignment_loss‚1.2184325482693794 /loss/FRt4/detection_loss‚2.1862233291165185 trainer/global_step‚4649 /loss/FBt3/assignment_loss‚0.260017750118974 1loss/SRqqt2/detection_loss‚2.1519578529728784 /loss/FBt3/detection_loss‚0.7137233417940466 /loss/FBt4/detection_loss‚3.2607227683067324 0loss/FRt3/assignment_loss‚1.7818262071713158 2loss/SRqqt2/assignment_loss‚1.5589726092086897 0loss/SRqqt1/detection_loss‚2.567193342372775 _runtime‚3669.423045233 /loss/FRt1/assignment_loss‚5.228520345687866 .loss/FRt2/assignment_loss‚4.33349426303591Ù’´YÓÓ 0loss/FRt3/assignment_loss‚1.7818262071713158 2loss/SRqqt2/assignment_loss‚1.5589726092086897 0loss/SRqqt1/detection_loss‚2.567193342372775 _runtime‚3669.423045233 /loss/FRt1/assignment_loss‚5.228520345687866 .loss/FRt2/assignment_loss‚4.33349426303591 /loss/FRt2/detection_loss‚2.1862923758370534 /loss/FRt3/detection_loss‚1.9342655090858107 0loss/SRqqt4/detection_loss‚1.598136268556118 _step‚380 2loss/SRqqt1/assignment_loss‚2.0383855141699314 2loss/SRqqt4/assignment_loss‚1.6526715335746607 &loss/total_loss‚50.533189647397045 epoch‚3 0loss/FRt4/assignment_loss‚3.5211879771489363 1loss/SRqqt3/assignment_loss‚1.281926613417454 1loss/SRqqt3/detection_loss‚1.3986825530300848 .loss/FBt1/detection_loss‚2.939144325256348 .loss/FBt2/assignment_loss‚0.46532531181374 .loss/FRt1/detection_loss‚3.176379151344299 /loss/FBt2/detection_loss‚2.2255507707595825 % _timestamp‚1.7884485539515333e+09 /loss/FBt1/assignment_loss‚0.854157385347935 0loss/FBt4/assignment_loss‚1.2184325482693794 /loss/FRt4/detection_loss‚2.1862233291165185 trainer/global_step‚4649 /loss/FBt3/assignment_loss‚0.260017750118974 1loss/SRqqt2/detection_loss‚2.1519578529728784 /loss/FBt3/detection_loss‚0.7137233417940466 /loss/FBt4/detection_loss‚3.2607227683067324üf¹øÿ“Ô‚(:ˆ ¯žæÔéщý gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚176.023' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.2046 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0\ãcSÕ‚(:È ¯žæÔÅøÿ( disk.nvme0c0n1.out‚14204.22802734375 disk.sda3.out‚ 140.640625 disk.drbd2266.out‚0 disk.rbd0.in‚0& proc.memory.availableMB‚ 545.078125 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚14204.22802734375# memory_percent‚96.6731071472168 disk.rbd1.out‚ 19.2412109375 disk.dm-1.out‚ 140.640625 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda1.out‚0* proc.memory.percent‚15.686726570129395) disk./.usagePercent‚20.18062002280392 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0$ disk./.usageGB‚300.7058219909668 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0 disk.sda1.in‚0 proc.cpu.threads‚17 disk.dm-1.in‚0# proc.memory.rssMB‚ 2570.11328125 cpu‚25.30300121666933 disk.sda2.in‚0& disk.nvme0n1.out‚14204.22802734375 disk.sda3.in‚0 disk.sda.out‚ 140.640625 network.sent‚ 1.0407138e+07 network.recv‚ 1.312684e+06 disk.drbd2080.in‚0ÅêzÖ‚(:ˆ ¾žæÔ›Œ—/ŠÆ}ý gpu.0.gpu‚41.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.806' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.7612 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0øpÍíWׂ(:Ì ¾žæÔ±Êûÿ disk.sda3.out‚ 141.2265625 disk.sda.out‚ 141.2265625 disk.rbd1.out‚ 19.2412109375 disk.nvme0n1.in‚0.078125( disk.nvme0c0n1.out‚14262.34521484375# disk.dm-0.out‚14262.34521484375 disk.dm-1.in‚0 disk.drbd2265.in‚0$ memory_percent‚96.67327404022217* proc.memory.percent‚15.686726570129395# disk./.usageGB‚300.732234954834 disk.drbd2266.in‚0 disk.sda1.in‚0 cpu‚25.301967499432497' disk./.usagePercent‚20.182392618971 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 141.2265625 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.drbd2080.out‚0 network.recv‚ 1.328279e+06 disk.rbd0.out‚0( proc.memory.availableMB‚ 545.05078125 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚0& disk.nvme0n1.out‚14262.34521484375# proc.memory.rssMB‚ 2570.11328125 network.sent‚ 1.0520467e+07¬ÓÑw‘Ø‚(:† ÍžæÔØèšý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚138.12' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚27.624 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0$†‹\Ù‚(:Ñ ÍžæÔÿÊŸÿ disk.dm-1.out‚ 142.05078125 disk.dm-0.in‚0.078125# disk.dm-0.out‚14310.30224609375 proc.cpu.threads‚17& disk.nvme0n1.out‚14310.30224609375 network.recv‚ 1.331408e+06 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.sda3.out‚ 142.05078125) disk./.usagePercent‚20.16679023979673 disk.drbd1716.out‚0 disk.drbd2265.out‚0% disk./.usageGB‚300.49974822998047 disk.sda2.in‚0 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚0( disk.nvme0c0n1.out‚14310.30224609375 disk.drbd2266.in‚0( proc.memory.availableMB‚ 544.76953125# proc.memory.rssMB‚ 2570.11328125* proc.memory.percent‚15.686726570129395 cpu‚25.30089844901428 disk.sda.in‚0 disk.sda.out‚ 142.05078125 disk.sda1.in‚0 disk.nvme0c0n1.in‚0.078125# memory_percent‚96.6749906539917 network.sent‚ 1.0525623e+07 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 19.3896484375 disk.drbd2080.out‚0 disk.drbd2266.out‚0«Àj2Ú- _wandbruntime‚3705  _runtime‚3705Ì·ž=ÇÛÁ & lr-AdamW/pg1‚9.715878108393766e-05 & lr-AdamW/pg2‚9.715878108393766e-05 trainer/global_step‚4699 % _timestamp‚1.7884485904254565e+09 _runtime‚3705.897073796 _step‚381ðŸC)ÌÜÆ % _timestamp‚1.7884485904254565e+09 _runtime‚3705.897073796 _step‚381 & lr-AdamW/pg1‚9.715878108393766e-05 & lr-AdamW/pg2‚9.715878108393766e-05 trainer/global_step‚4699ýP2Ý- _wandbruntime‚3705  _runtime‚3705/~Ü'ÞÖO ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚301051500241408 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚67925130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61954018 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚275758527253504 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3705.898064265 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67925132 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3921474741760 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67925130 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67925132 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚88449229 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚264735634603008 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3922131104256 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39624556 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚271837052511744 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22329910 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚296937635964416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚271837052511744 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3921474741760 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚275758527253504 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚296937612024320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚275759201618944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39624558 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22329462X. á'ßÛO TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚67925132 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3921474741760 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚67925130 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚67925132 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚88449229 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚264735634603008 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3922131104256 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚39624556 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚271837052511744 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22329910 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚296937635964416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚271837052511744 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3921474741760 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚275758527253504 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚296937612024320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚275759201618944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚39624558 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22329462 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚301051500241408 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚67925130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚61954018 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚275758527253504 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3705.898064265 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0þd‹].2à- _wandbruntime‚3706  _runtime‚3706$¿s"ááM D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 156386791 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3922950545408 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚88467709 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22334363 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚438 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚296999286702080 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4114727153152 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚39632726 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 156406857 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚275816374098944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚296999262761984 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 156406857 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67939148 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚4699 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚3706.589159895 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚88467709 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚264790538733568 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61967089 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚301114013855232 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _step‚383 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚440 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39632724 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3922950545408 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚264790521694208 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚275816374098944 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4114565639680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3908514185874 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3907859747910 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚268698381442118 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 156386791 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22333925 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67939148 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61966649 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚271893405550592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚268699052919442 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884485911170824e+09 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚67939148 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚61966649 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚271893405550592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚268699052919442 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884485911170824e+09 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 156406857 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚67939148 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚4699 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚3706.589159895 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚88467709 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚264790538733568 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚61967089 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚301114013855232 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _step‚383 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚440 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚39632724 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3922950545408 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚264790521694208 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚275816374098944 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4114565639680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3908514185874 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3907859747910 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚268698381442118 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 156386791 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22333925ÿüÄd2ã- _wandbruntime‚3706  _runtime‚3706(ts#UäÏ 1loss/SRqqt3/assignment_loss‚1.120540550729232 0loss/SRqqt1/detection_loss‚2.625019509946146 1loss/SRqqt2/detection_loss‚2.2513534146838072 _runtime‚3706.590144122 %loss/total_loss‚48.98220311062581 epoch‚3 2loss/SRqqt2/assignment_loss‚1.4642757831550226 2loss/SRqqt4/assignment_loss‚1.4703288047478116 /loss/FRt1/detection_loss‚3.3110991716384888 0loss/FRt1/assignment_loss‚4.5351408672332765 1loss/SRqqt1/assignment_loss‚2.425029150901302 1loss/FBt4/assignment_loss‚0.28748033709254944 /loss/FRt4/detection_loss‚2.1859155816060527 0loss/FRt3/assignment_loss‚2.3199974389399514 .loss/FRt4/assignment_loss‚3.33636541278274 .loss/FBt4/detection_loss‚3.507659968402651 trainer/global_step‚4699 0loss/FRt2/assignment_loss‚3.9339125813270104 0loss/FBt1/assignment_loss‚0.3976171667472061 0loss/FBt3/assignment_loss‚0.2794806840301407 /loss/FRt3/detection_loss‚1.8019771812073255 1loss/SRqqt4/detection_loss‚1.5405626040080498 -loss/FBt2/detection_loss‚2.05374282949111 /loss/FBt3/detection_loss‚0.9207105628130111 $ _timestamp‚1.788448591118624e+09 .loss/FRt2/detection_loss‚2.253135698182242 /loss/FBt1/detection_loss‚2.9795683196612766 _step‚384 0loss/FBt2/assignment_loss‚0.5902836439743302 0loss/SRqqt3/detection_loss‚1.391005847325076ÍÖ9ZåÔ /loss/FBt3/detection_loss‚0.9207105628130111 $ _timestamp‚1.788448591118624e+09 .loss/FRt2/detection_loss‚2.253135698182242 /loss/FBt1/detection_loss‚2.9795683196612766 _step‚384 0loss/FBt2/assignment_loss‚0.5902836439743302 0loss/SRqqt3/detection_loss‚1.391005847325076 1loss/SRqqt3/assignment_loss‚1.120540550729232 0loss/SRqqt1/detection_loss‚2.625019509946146 1loss/SRqqt2/detection_loss‚2.2513534146838072 _runtime‚3706.590144122 %loss/total_loss‚48.98220311062581 epoch‚3 2loss/SRqqt2/assignment_loss‚1.4642757831550226 2loss/SRqqt4/assignment_loss‚1.4703288047478116 /loss/FRt1/detection_loss‚3.3110991716384888 0loss/FRt1/assignment_loss‚4.5351408672332765 1loss/SRqqt1/assignment_loss‚2.425029150901302 1loss/FBt4/assignment_loss‚0.28748033709254944 /loss/FRt4/detection_loss‚2.1859155816060527 0loss/FRt3/assignment_loss‚2.3199974389399514 .loss/FRt4/assignment_loss‚3.33636541278274 .loss/FBt4/detection_loss‚3.507659968402651 trainer/global_step‚4699 0loss/FRt2/assignment_loss‚3.9339125813270104 0loss/FBt1/assignment_loss‚0.3976171667472061 0loss/FBt3/assignment_loss‚0.2794806840301407 /loss/FRt3/detection_loss‚1.8019771812073255 1loss/SRqqt4/detection_loss‚1.5405626040080498 -loss/FBt2/detection_loss‚2.05374282949111€PßÓæ‚(:ˆ ÜžæÔü‘þü gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.766' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5532 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÆeQç‚(:Æ ÜžæÔÔì ÿ disk.rbd1.in‚0 disk.drbd2080.out‚0" disk.dm-0.out‚14368.4150390625 disk.rbd1.out‚ 19.3896484375" memory_percent‚96.673583984375 proc.memory.availableMB‚545 cpu‚25.299894503208282 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚0 disk.drbd1716.out‚0# proc.memory.rssMB‚ 2570.11328125 proc.cpu.threads‚17% disk./.usageGB‚300.55984115600586 disk.rbd0.out‚0 disk.sda3.in‚0% disk.nvme0n1.out‚14368.4150390625 disk.dm-1.out‚ 142.55078125 disk.drbd2265.in‚0 network.sent‚ 1.0532736e+07 network.recv‚ 1.336797e+06* disk./.usagePercent‚20.170823126483615 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚14368.4150390625 disk.drbd2266.in‚0 disk.sda.out‚ 142.55078125 disk.drbd1716.in‚0* proc.memory.percent‚15.686726570129395 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda3.out‚ 142.55078125 disk.drbd2266.out‚0 disk.sda2.out‚0ÅÃökžè‚(:“ ëžæÔ¡Åþý gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.017' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.203399999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0/f·¢Jé‚(:¿ ëžæÔüÑ”€ disk.dm-1.out‚ 143.0546875 disk.sda.in‚0 disk.sda.out‚ 143.0546875 disk.sda2.out‚0 network.recv‚ 1.34267e+06$ disk./.usageGB‚300.5864562988281 disk.drbd2265.in‚0# proc.memory.rssMB‚ 2570.11328125 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚14413.4736328125 disk.sda3.in‚0 disk.drbd2266.out‚0 cpu‚25.298819496973145 network.sent‚ 1.064036e+07 disk.drbd1716.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0" memory_percent‚96.673583984375 proc.memory.availableMB‚545* proc.memory.percent‚15.686726570129395 disk.drbd2080.in‚0 disk.sda3.out‚ 143.0546875 disk.rbd1.out‚ 19.5615234375 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚14413.4736328125 disk.dm-1.in‚0 proc.cpu.threads‚17 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125" disk.dm-0.out‚14413.4736328125 disk.sda1.in‚0 disk.sda1.out‚0) disk./.usagePercent‚20.17260929105001 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0âPnu2ê- _wandbruntime‚3742  _runtime‚3742>‘îÇëÁ trainer/global_step‚4749 % _timestamp‚1.7884486272701235e+09 _runtime‚3742.741733058 _step‚385 & lr-AdamW/pg1‚9.638206785424639e-05 & lr-AdamW/pg2‚9.638206785424639e-05:v§ÐÌìÆ % _timestamp‚1.7884486272701235e+09 _runtime‚3742.741733058 _step‚385 & lr-AdamW/pg1‚9.638206785424639e-05 & lr-AdamW/pg2‚9.638206785424639e-05 trainer/global_step‚4749‘o2í- _wandbruntime‚3742  _runtime‚3742üûq5Ý'î×O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3948533090110 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _step‚386 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚68625932 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚300020148907520 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚271429374221118 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚274654704454144 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚447 _runtime‚3742.742732945 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚274654722457088 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚271428702744002 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3963116910080 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3962460547584 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚68625932 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚89355243 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚89375281 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4156646299136 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚4749 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40032958 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22556241 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3947878652354 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚278617165001728 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22556686 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚304176980673536 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚274654722457088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62589197 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚278617839367168 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62589644 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚89355243 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40032956 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚267480824091648 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68625930 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚278617165001728 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3962460547584 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884486272709315e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚274654704454144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3963116910080 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 157981173 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚278617839367168 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚304176795206656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 158001213 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚300020172847616 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚267480841131008 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40032956 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚267480824091648 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚68625930 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚278617165001728 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3962460547584 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884486272709315e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚274654704454144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3963116910080 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 157981173 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚278617839367168 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚304176795206656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 158001213 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.Ôlm¹šallocated‚300020172847616 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚267480841131008 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22556686 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚304176980673536 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚274654722457088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚62589197 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚278617839367168 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚62589644 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚89355243 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0‚6²—›2ð- _wandbruntime‚3743  _runtime‚3743<ïlÑç&ñáM EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 158033709 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3949349525282 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚274711057492992 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22561155 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚304239290335232 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚271484423309526 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3948695087318 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200133632 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚278674337468416 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚278675011847168 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚274711075495936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3963936351232 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚300081823585280 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚267535728222208 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚274711075495936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68639946 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3963279975424 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 158033709 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68639948 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3963279975424 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40041126 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚3743.434416307 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚304239490468864 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62601824 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 158013643 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 158013643 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68639948 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚89393761 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚278675011847168 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22560700 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚89373697 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62602281 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚387 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4157490690048 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚271485094786850 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884486279623928e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚89373697 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 158033709 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3949349525282 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚274711057492992 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22561155 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚304239290335232 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚271484423309526 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3948695087318 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200133632 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚278674337468416 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚278675011847168 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚274711075495936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3963936351232 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚300081823585280 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚267535728222208 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚274711075495936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚68639946 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3963279975424 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 158033709 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚68639948 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3963279975424 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40041126 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚3743.434416307 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚304239490468864 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚62601824 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 158013643 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 158013643 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚68639948 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚89393761 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚278675011847168 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚22560700 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚89373697 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚62602281 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚387 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4157490690048 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚271485094786850 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884486279623928e+09ƒBÅÑ2ó- _wandbruntime‚3743  _runtime‚3743Ÿyñ}VôÐ /loss/FRt4/assignment_loss‚3.094115738164295 0loss/SRqqt2/detection_loss‚1.769964964322324 0loss/SRqqt3/detection_loss‚1.538515335642824 /loss/FRt1/assignment_loss‚4.544776017849262 0loss/FRt3/assignment_loss‚1.5795403969287873 1loss/SRqqt3/assignment_loss‚1.061804134179564 trainer/global_step‚4749 2loss/SRqqt4/assignment_loss‚1.3829290563942955 0loss/SRqqt1/detection_loss‚2.900972734604563 2loss/SRqqt1/assignment_loss‚2.5317229216120074 1loss/SRqqt2/assignment_loss‚1.611775430622247 1loss/FBt1/assignment_loss‚0.24543918445141344 /loss/FRt2/detection_loss‚2.1015193812679827 %loss/total_loss‚48.28270599302975 0loss/FRt2/assignment_loss‚3.3440075474872923 0loss/FBt2/assignment_loss‚0.5402211756259021 1loss/FBt3/assignment_loss‚0.17193540932774473 /loss/FRt4/detection_loss‚2.2046422607858074 /loss/FBt2/detection_loss‚2.7820265665650368 1loss/SRqqt4/detection_loss‚1.6235605009266587 .loss/FBt1/detection_loss‚2.935493502765894 /loss/FBt3/detection_loss‚0.9774315098673105 _step‚388 1loss/FBt4/assignment_loss‚0.34843949213973247 /loss/FRt3/detection_loss‚1.9029194370657205 .loss/FBt4/detection_loss‚3.952495828270912 _runtime‚3743.435558824 .loss/FRt1/detection_loss‚3.136457466162168 epoch‚3 # _timestamp‚1.78844862796404e+09–J:d[õÕ 0loss/FRt3/assignment_loss‚1.5795403969287873 1loss/SRqqt3/assignment_loss‚1.061804134179564 trainer/global_step‚4749 2loss/SRqqt4/assignment_loss‚1.3829290563942955 0loss/SRqqt1/detection_loss‚2.900972734604563 2loss/SRqqt1/assignment_loss‚2.5317229216120074 1loss/SRqqt2/assignment_loss‚1.611775430622247 1loss/FBt1/assignment_loss‚0.24543918445141344 /loss/FRt2/detection_loss‚2.1015193812679827 %loss/total_loss‚48.28270599302975 0loss/FRt2/assignment_loss‚3.3440075474872923 0loss/FBt2/assignment_loss‚0.5402211756259021 1loss/FBt3/assignment_loss‚0.17193540932774473 /loss/FRt4/detection_loss‚2.2046422607858074 /loss/FBt2/detection_loss‚2.7820265665650368 1loss/SRqqt4/detection_loss‚1.6235605009266587 .loss/FBt1/detection_loss‚2.935493502765894 /loss/FBt3/detection_loss‚0.9774315098673105 _step‚388 1loss/FBt4/assignment_loss‚0.34843949213973247 /loss/FRt3/detection_loss‚1.9029194370657205 .loss/FBt4/detection_loss‚3.952495828270912 _runtime‚3743.435558824 .loss/FRt1/detection_loss‚3.136457466162168 epoch‚3 # _timestamp‚1.78844862796404e+09 /loss/FRt4/assignment_loss‚3.094115738164295 0loss/SRqqt2/detection_loss‚1.769964964322324 0loss/SRqqt3/detection_loss‚1.538515335642824 /loss/FRt1/assignment_loss‚4.544776017849262„Õì=“ö‚(:ˆ úžæÔȨ‘þ gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.442' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6884 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0›ÍÎMY÷‚(:Î úžæÔ§®ñþ network.recv‚ 1.367341e+06 disk.drbd2265.in‚0 disk.sda2.out‚0% disk.nvme0n1.out‚14478.2451171875 disk.dm-1.out‚ 143.91015625 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.sda.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0$ memory_percent‚96.67539596557617 network.sent‚ 1.0656618e+07$ disk./.usageGB‚300.6077537536621 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.sda3.out‚ 143.91015625 disk.rbd0.in‚0" disk.dm-0.out‚14478.2451171875 disk.sda1.out‚0& proc.memory.availableMB‚ 544.703125# proc.memory.rssMB‚ 2570.11328125* proc.memory.percent‚15.686726570129395 cpu‚25.297849796006798* disk./.usagePercent‚20.174038581113674 disk.drbd2265.out‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 proc.cpu.threads‚17 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚14478.2451171875 disk.rbd1.out‚ 19.5615234375 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda.out‚ 143.91015625„(Êæ“ø‚(:ˆ ‰ŸæÔ‹œ‚ý gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚134.709' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.9418 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0&¶ëxXù‚(:Í ‰ŸæÔÙ·’€ disk.rbd1.out‚ 19.6240234375 disk.drbd1716.out‚0* proc.memory.percent‚15.686726570129395 disk.rbd1.in‚0 disk.sda3.in‚0& proc.memory.availableMB‚ 544.953125 disk.sda.in‚0% disk.nvme0n1.out‚14529.1083984375 disk.drbd1716.in‚0 disk.drbd2265.in‚0$ memory_percent‚96.67387008666992 disk.nvme0c0n1.in‚0.078125) disk./.usagePercent‚20.17503649848214$ disk./.usageGB‚300.6226234436035 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.sda3.out‚ 144.40234375 disk.sda.out‚ 144.40234375' disk.nvme0c0n1.out‚14529.1083984375 disk.drbd2266.out‚0# proc.memory.rssMB‚ 2570.11328125 cpu‚25.296852630641208 disk.rbd0.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.out‚ 144.40234375 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 proc.cpu.threads‚17 network.sent‚ 1.0764269e+07 network.recv‚ 1.372588e+06 disk.rbd0.out‚0" disk.dm-0.out‚14529.1083984375OB2ú- _wandbruntime‚3779  _runtime‚3779ILÍfÇûÁ _runtime‚3779.790041938 _step‚389 & lr-AdamW/pg1‚9.559846930031973e-05 & lr-AdamW/pg2‚9.559846930031973e-05 trainer/global_step‚4799 % _timestamp‚1.7884486643182943e+09Dû=µÌüÆ _runtime‚3779.790041938 _step‚389 & lr-AdamW/pg1‚9.559846930031973e-05 & lr-AdamW/pg2‚9.559846930031973e-05 trainer/global_step‚4799 % _timestamp‚1.7884486643182943e+09…´Þ(2ý- _wandbruntime‚3779  _runtime‚3779pçü @þ×O H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22783943 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22784375 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.4¡» #on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚277472374399488 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4004102715904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚277472374399488 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4004102715904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40441358 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚274214744611410 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4199626081280 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4199789705216 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚307302311872000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚4799 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚90281295 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚303102709730816 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚270226047659008 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3989368429518 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4003446353408 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 159628065 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚432 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚281475802749952 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚390 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚90301333 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40441356 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 159608025 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4003446353408 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚307302499436032 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚277472356396544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚434 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚281476477115392 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚90301333 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚3779.790890509 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69326732 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚281475802749952 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2æ’ߨâ'ÿÜO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚277472356396544 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚303102685790720 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3988713991762 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚281476477115392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚390 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚90301333 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 262745088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚40441356 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 159608025 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4003446353408 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚307302499436032 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚277472356396544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚434 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚281476477115392 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚90301333 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚3779.790890509 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚69326732 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚281475802749952 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚22783943 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚22784375 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚277472374399488 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4004102715904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚277472374399488 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4004102715904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚40441358 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚274214744611410 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4199626081280 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4199789705216 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚307302311872000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚4799 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚90281295 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚303102709730816 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚270226047659008 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3989368429518 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4003446353408 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 159628065 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚432 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚281475802749952 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚270280934750208 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 159660561 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚281532975216640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚63237945 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22788854 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚281533649595392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69340746 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4004922157056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚277528709435392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40449526 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚274271136654258 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚274270465176934 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3989530426726 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4004265781248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚307364824871424 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 159640495 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _step‚391 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 trainer/global_step‚4799 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚270280951789568 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884486650132227e+09 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _runtime‚3780.485318287 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4004922157056 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69340746 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 159660561 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚90299749 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4200488343040 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚90319813 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚303164336528384 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3990184864690 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69340748 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69340748 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚281533649595392 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚303164360468480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964¼¯ZÕL„æM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚90319813 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚303164336528384 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3990184864690 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚69340748 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚69340748 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚281533649595392 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚303164360468480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4004265781248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚63238380 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚307365014519296 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 159640495 ;4DeviceStatsMonito!ýÌ r.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚270280934750208 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 159660561 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚281532975216640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚63237945 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚22788854 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚281533649595392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚69340746 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4004922157056 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚277528709435392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚40449526 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 262745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚274271136654258 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚274270465176934 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3989530426726 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4004265781248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚307364824871424 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 159640495 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _step‚391 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 trainer/global_step‚4799 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚270280951789568 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884486650132227e+09 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _runtime‚3780.485318287 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4004922157056 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚69340746 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 159660561 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚90299749 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4200488343040 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088‡6€N©2…- _wandbruntime‚3780  _runtime‚3780±ÝvÓQ†Ë 1loss/SRqqt4/assignment_loss‚1.636099828042738 /loss/FRt1/detection_loss‚2.9528312472736133 /loss/FRt4/assignment_loss‚3.296381996870041 0loss/FBt1/assignment_loss‚0.5467867987711613 /loss/FBt3/detection_loss‚0.8766742768854677 1loss/SRqqt2/detection_loss‚2.2459648236250267 1loss/SRqqt3/detection_loss‚1.4478320758494119 _runtime‚3780.486567845 epoch‚3 trainer/global_step‚4799 /loss/FRt3/assignment_loss‚2.410267472267151 0loss/SRqqt1/assignment_loss‚2.54394967854023 2loss/SRqqt2/assignment_loss‚1.5013213321471062 2loss/SRqqt3/assignment_loss‚1.3375165083460392 -loss/FRt2/detection_loss‚2.01722806804585 0loss/SRqqt4/detection_loss‚1.857971060535182 /loss/FRt1/assignment_loss‚4.805102425463059 0loss/FBt4/assignment_loss‚0.4614388226327719 /loss/FRt3/detection_loss‚1.8050041921712734 /loss/FBt2/detection_loss‚2.9097173511981964 $loss/total_loss‚50.4572819935169 $ _timestamp‚1.788448665014814e+09 _step‚392 /loss/FRt4/detection_loss‚2.2842948669195176 .loss/FBt2/assignment_loss‚0.59484038984765 1loss/FBt3/assignment_loss‚0.22110268462184143 /loss/FBt1/detection_loss‚2.9172102488004246 /loss/FRt2/assignment_loss‚3.872104676264637 0loss/SRqqt1/detection_loss‚2.737927334649222 /loss/FBt4/detection_loss‚3.1777138337492943•£˜ V‡Ð 1loss/SRqqt2/detection_loss‚2.2459648236250267 1loss/SRqqt3/detection_loss‚1.4478320758494119 _runtime‚3780.486567845 epoch‚3 trainer/global_step‚4799 /loss/FRt3/assignment_loss‚2.410267472267151 0loss/SRqqt1/assignment_loss‚2.54394967854023 2loss/SRqqt2/assignment_loss‚1.5013213321471062 2loss/SRqqt3/assignment_loss‚1.3375165083460392 -loss/FRt2/detection_loss‚2.01722806804585 0loss/SRqqt4/detection_loss‚1.857971060535182 /loss/FRt1/assignment_loss‚4.805102425463059 0loss/FBt4/assignment_loss‚0.4614388226327719 /loss/FRt3/detection_loss‚1.8050041921712734 /loss/FBt2/detection_loss‚2.9097173511981964 $loss/total_loss‚50.4572819935169 $ _timestamp‚1.788448665014814e+09 _step‚392 /loss/FRt4/detection_loss‚2.2842948669195176 .loss/FBt2/assignment_loss‚0.59484038984765 1loss/FBt3/assignment_loss‚0.22110268462184143 /loss/FBt1/detection_loss‚2.9172102488004246 /loss/FRt2/assignment_loss‚3.872104676264637 0loss/SRqqt1/detection_loss‚2.737927334649222 /loss/FBt4/detection_loss‚3.1777138337492943 1loss/SRqqt4/assignment_loss‚1.636099828042738 /loss/FRt1/detection_loss‚2.9528312472736133 /loss/FRt4/assignment_loss‚3.296381996870041 0loss/FBt1/assignment_loss‚0.5467867987711613 /loss/FBt3/detection_loss‚0.8766742768854677ˆO´ÆÏ“ˆ‚(:ˆ §ŸæÔ”ˆý gpu.0.gpu‚45.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚175.734' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.1468 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ã;¡X‰‚(:Í §ŸæÔ¿¡þÿ network.sent‚ 1.0877036e+07 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125& disk.nvme0n1.out‚14627.71142578125 disk.drbd2265.in‚0 disk.sda1.in‚0$ disk./.usageGB‚300.6573486328125 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚14627.71142578125 disk.drbd2265.out‚0( proc.memory.availableMB‚ 544.62109375 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.drbd1716.out‚0* proc.memory.percent‚15.686726570129395 cpu‚25.29485295073113 network.recv‚ 1.402927e+06 proc.cpu.threads‚17 disk.sda.out‚ 145.63671875) disk./.usagePercent‚20.17736693506633 disk.drbd2080.in‚0 disk.sda3.in‚0 disk.drbd2266.in‚0# disk.dm-0.out‚14627.71142578125# proc.memory.rssMB‚ 2570.11328125 disk.sda3.out‚ 145.63671875 disk.sda.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 145.63671875 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 19.765625 disk.drbd1716.in‚0$ memory_percent‚96.67589664459229iÈBŠ‚(:’ ¶ŸæÔƒ‡„ý gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚123.266' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.653200000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0£#‡R‹‚(:Ç ¶ŸæÔûñþ disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd0.out‚0 disk.sda.in‚0& disk.nvme0n1.out‚14681.28076171875" proc.memory.rssMB‚ 2650.8984375* proc.memory.percent‚16.179800033569336 disk.drbd1716.in‚0 network.recv‚ 1.409498e+06$ disk./.usageGB‚300.6801414489746 disk.sda3.out‚ 146.265625 disk.drbd2266.out‚0 disk.sda3.in‚0 disk.drbd2266.in‚0 disk.sda2.in‚0$ memory_percent‚97.17271327972412 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 network.sent‚ 1.097532e+07( disk.nvme0c0n1.out‚14681.28076171875 disk.dm-1.in‚0 disk.drbd1716.out‚0( proc.memory.availableMB‚ 463.22265625 cpu‚25.297557795473146 disk.rbd0.in‚0 disk.rbd1.out‚ 19.765625 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 146.265625 disk.drbd2080.in‚0# disk.dm-0.out‚14681.28076171875 disk.sda.out‚ 146.265625 disk.drbd2265.out‚0 disk.drbd2080.out‚0 proc.cpu.threads‚17* disk./.usagePercent‚20.1788965800834186)D¹Œ‚(:’ ÅŸæÔ§±¥ý gpu.0.gpu‚17.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚197.362' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚39.47239999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ý3úýU‚(:Ê ÅŸæԪоÿ disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0& disk.nvme0n1.out‚14728.28857421875% disk./.usageGB‚300.68177795410156 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 146.8359375 disk.rbd1.out‚ 19.828125 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2080.in‚0 network.sent‚ 1.0979583e+07* disk./.usagePercent‚20.179006407315605 disk.dm-1.out‚ 146.8359375 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125$ memory_percent‚97.13778495788574' proc.memory.availableMB‚ 468.9453125 disk.sda1.out‚0 disk.dm-1.in‚0* proc.memory.percent‚16.138362884521484 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.drbd1716.in‚0! proc.memory.rssMB‚ 2644.109375# disk.dm-0.out‚14728.28857421875 disk.sda1.in‚0 disk.sda3.out‚ 146.8359375 cpu‚25.314942598534415( disk.nvme0c0n1.out‚14728.28857421875 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda3.in‚0 network.recv‚ 1.412096e+06 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda.in‚0l+ú“Ž‚(:ˆ ÔŸæÔÒ¬Üü gpu.0.gpu‚56.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚126.081' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.2162 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¡ÝN‚(:à ԟæÔÎÿ‚€% disk.nvme0c0n1.out‚14779.87890625 disk.sda3.in‚0 disk.sda3.out‚ 147.62109375 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.rbd0.out‚0 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.dm-0.out‚14779.87890625 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚0 disk.sda1.in‚0 disk.rbd1.out‚ 19.828125 disk.nvme0c0n1.in‚0.078125 network.sent‚ 1.0985991e+07 network.recv‚ 1.417247e+06$ disk./.usageGB‚300.6786804199219 disk.drbd2080.in‚0 disk.drbd2266.out‚0# disk.nvme0n1.out‚14779.87890625! proc.memory.rssMB‚ 2644.109375 disk.dm-1.out‚ 147.62109375 disk.sda2.out‚0# memory_percent‚97.1411943435669* disk./.usagePercent‚20.178798529197802 disk.drbd2265.in‚0 disk.sda.out‚ 147.62109375 disk.rbd1.in‚0* proc.memory.percent‚16.138362884521484 cpu‚25.333615246446993 disk.dm-1.in‚0 disk.sda2.in‚0( proc.memory.availableMB‚ 468.38671875 proc.cpu.threads‚17e'&‚(:’ ãŸæÔ“Ë‹ý gpu.0.gpu‚9.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.969' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.593799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0baí÷N‘‚(:à ãŸæÔˆ¨ìþ' proc.memory.availableMB‚ 468.9609375 disk.drbd2080.in‚0 network.recv‚ 1.420655e+06 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.drbd1716.out‚0 disk.sda3.in‚0% disk./.usageGB‚300.69457626342773 disk.sda.in‚0 disk.sda.out‚ 148.25390625 disk.rbd1.out‚ 19.9697265625" disk.nvme0n1.out‚ 14825.3359375 disk.drbd1716.in‚0 disk.drbd2265.in‚0* proc.memory.percent‚16.138362884521484 disk.rbd0.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125! proc.memory.rssMB‚ 2644.109375 cpu‚25.351201101079198 disk.sda1.in‚0 disk.sda2.out‚0 proc.cpu.threads‚17 network.sent‚ 1.0991157e+07 disk.rbd0.out‚0 disk.sda2.in‚0 disk.dm-1.in‚0$ disk.nvme0c0n1.out‚ 14825.3359375 disk.sda3.out‚ 148.25390625 disk.drbd2266.in‚0 disk.dm-1.out‚ 148.25390625# memory_percent‚97.1376895904541) disk./.usagePercent‚20.17986531259298 disk.dm-0.in‚0.078125 disk.dm-0.out‚ 14825.3359375 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0Cº2’- _wandbruntime‚3855  _runtime‚3855¿3C*“½T E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚278023803484678 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚285413932557824 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚285414443390976 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚281362090245120 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚23175444 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510833152 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15977 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9955586048 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15979 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚285413932557824 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 YLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚70374818 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 4249740881408 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 4249625537536 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 OIDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.current‚0 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 4052335142912 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 [GDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated‚285414443390976 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚281362108248064 UHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚91327028 QGDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak‚24128 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚41062270 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15979 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 161717823 FDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 _step‚393 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚70374816 VIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚64237866 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚311489438847488 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚278024311438430 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 trainer/global_step‚4835 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 161701844 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚307239674025984 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.freed‚0 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 4037416733790 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚91343005 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914392 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚154 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 139283968 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚152 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 QDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚70374816 QDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚91327028 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944ùvp ”ÂT WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚285413932557824 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 YLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚70374818 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 4249740881408 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 4249625537536 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 OIDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.current‚0 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 4052335142912 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 [GDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated‚285414443390976 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚281362108248064 UHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚91327028 QGDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak‚24128 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚41062270 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15979 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 161717823 FDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 _step‚393 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚70374816 VIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚64237866 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚311489438847488 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚278024311438430 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 trainer/global_step‚4835 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 161701844 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚307239674025984 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.freed‚0 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 336208384 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 4037416733790 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚91343005 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914392 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚154 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 139283968 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚152 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 QDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚70374816 QDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚91327028 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚278023803484678 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚285413932557824 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚285414443390976 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚281362090245120 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚23175444 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510833152 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15977 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9955586048 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15979‰ÈË–2•- _wandbruntime‚3856  _runtime‚3856k½ð€N)–ÈR YJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 ^LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚ 4052700269568 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 161735931 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 SFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚70385622 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚23180144 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 161735931 PCDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed‚64248764 trainer/global_step‚4835 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 WJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚91350307 _KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.freed‚274022171424768 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 BDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 OBDeviceStatsMonitor.on_validation_batch_end/active.small_pool.freed‚91334330 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 MEDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.current‚154 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚41068772 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚278059968807326 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 SFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚70385624 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚ 4052207439360 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 115343872 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914392 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 WJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚70385624 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 TFDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.current‚ 510833152 SFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚91334330 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚41068774 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 ]KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚ 4037289428806 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884487407897468e+09 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚281398920273920 bPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚ 4249990664192 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚281398920273920 XIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 MGDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 _step‚394 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24401 SFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚91350307 QCDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.current‚ 510833152 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 336208384 JDDeviceStatsMonitor.on_validation_batch_end/oversize_segments.current‚0 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 aODeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.allocated‚ 4037780343198 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚285451638546432 \HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated‚285451638546432 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 KADeviceStatsMonitor.on_validation_batch_end/active.small_pool.peak‚24128GòiS)—ÍR XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 SFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚70385622 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚23180144 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 161735931 PCDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed‚64248764 trainer/global_step‚4835 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 WJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚91350307 _KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.freed‚274022171424768 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 BDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 OBDeviceStatsMonitor.on_validation_batch_end/active.small_pool.freed‚91334330 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 MEDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.current‚154 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚41068772 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚278059968807326 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 SFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚70385624 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚ 4052207439360 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 115343872 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914392 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 WJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚70385624 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 TFDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.current‚ 510833152 SFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚91334330 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚41068774 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 ]KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚ 4037289428806 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884487407897468e+09 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚281398920273920 bPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚ 4249990664192 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚281398920273920 XIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 MGDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 _step‚394 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24401 SFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚91350307 QCDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.current‚ 510833152 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 336208384 JDDeviceStatsMonitor.on_validation_batch_end/oversize_segments.current‚0 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 aODeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.allocated‚ 4037780343198 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚285451638546432 \HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated‚285451638546432 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 KADeviceStatsMonitor.on_validation_batch_end/active.small_pool.peak‚24128 YJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 ^LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚ 4052700269568 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 161735931Š ˆ7F2˜- _wandbruntime‚3868  _runtime‚3868“ôq–Œ™Ò¦ 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5301687121391296 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3392690420150757 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.18827159702777863 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5900925993919373 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5271347761154175 +jet/accuracy_2_of_3‚0.24027033150196075 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.03394412249326706 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9877298474311829 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.43969300389289856 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.4343397319316864 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.06594319641590118 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 ,particle/sensitivity‚0.28862056136131287 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5526315569877625 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.4354199767112732 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.1603776216506958 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.9333333373069763 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.2020781934261322 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_4‚0.9013838171958923 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5594624876976013 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.03186967596411705 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0014430014416575432 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.014852607622742653 particle/accuracy_5_of_5‚1 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.05263157933950424 ("Purity/0FRt0SRqqt4FBt/event_purity‚0 ("Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5463369488716125 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5457584261894226 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 jet/accuracy_2_of_7‚0.5 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.1907934546470642 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9769506454467773 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.10111435502767563 particle/accuracy_1_of_2‚1 jet/accuracy_5_of_7‚0 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.06382978707551956 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5380474925041199 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5023148059844971 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9464285969734192 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.4814814329147339 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5484960675239563 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ,jet/accuracy_3_of_4‚0.047013767063617706 particle/accuracy_2_of_5‚1 particle/accuracy_4_of_5‚1 :"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.44999998807907104 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.39417287707328796 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 +jet/accuracy_3_of_5‚0.08395417034626007 ,&Purity/0FRt3SRqqt3FBt/event_proporËbÆùtion‚0 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9851285219192505 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.5642702579498291 9"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.4426426291465759 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 & Purity/*FRt1SRqqt4FBt/FRt_purity‚1 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.038266781717538834 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.48891738057136536 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.011798255145549774 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.1740759313106537 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9433333873748779 ) Purity/4FRt*SRqqt0FBt/FRt_purity‚0.25 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5124761462211609 + Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9375 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4583333730697632 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3819444477558136 particle/accuracy_3_of_6‚1 +"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.75 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.09645634144544601 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ("Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0 particle/accuracy_2_of_6‚1 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9980742335319519 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 6 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.991583526134491 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3673110902309418 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.15526314079761505 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.17052695155143738 7 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.1982758641242981 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1191326379776001 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5614035129547119 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5624845027923584 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5535714626312256 6 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.995573878288269 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 :"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.43947139382362366 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.14837120473384857 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.3362351059913635 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 , Purity/1FRt3SRqqt1FBt/FRt_purity‚0.40625 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 particle/accuracy_2_of_7‚1 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8140152096748352 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.15909090638160706 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 6 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.529411792755127 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.21938589215278625 )jet/accuracy_2_of_4‚0.337873637676239 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.20000000298023224 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.1666666716337204 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.17830407619476318 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.6666666269302368 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 -jet/accuracy_4_of_5‚0.0049923197366297245 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.1981857717037201 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.4799640476703644 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.06841396540403366 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9694769978523254 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9907218813896179 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.32567256689071655 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.48848021030426025 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.08953871577978134 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9848309755325317 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.2902558147907257 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4148467481136322 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.006998387165367603 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.3229166567325592 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 jet/accuracy_4_of_7‚0 particle/accuracy_4_of_7‚1 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.23062168061733246 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9886022806167603 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.14837120473384857 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 particle/accuracy_1_of_4‚1 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13488417863845825 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.3580247163772583 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.12574775516986847 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8688279986381531 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9270833134651184 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5385416746139526 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13656951487064362 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ( Purity/4FRt1SRqqt*FBt/FRt_purity‚0.5 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.02421383559703827 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.1496952325105667 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.6969696879386902 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5179523229598999 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 particle/accuracy_1_of_6‚1 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.4838888645172119 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 /"Purity/0FRt0SRqqt3FBt/event_purity‚0.078125 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 jet/accuracy_1_of_7‚1 particle/accuracy_3_of_7‚1 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5833333134651184 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0020674937404692173 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5141317844390869 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 :"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.49294808506965637 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4242689907550812 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 _runtime‚3868.894628462 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9595686197280884 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.0797727033495903 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0037707581650465727 -jet/accuracy_4_of_4‚0.0038580247201025486 particle/accuracy_1_of_1‚1 particle/accuracy_2_of_4‚1 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.019674401730298996 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5585334897041321 particle/accuracy_6_of_6‚1 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.43758171796798706 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3333333432674408 jet/accuracy_7_of_7‚0 +"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.75 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.1111111119389534 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.4234495759010315 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.36369088292121887 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14552029967308044 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9659090638160706 7 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.8571428656578064 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.15698209404945374 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9761709570884705 9"Purity/1FRt1SRqqt0FBt/event_purity‚0.0141415074467659 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.33124998211860657 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5565476417541504 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 ("Purity/*FRt0SRqqt4FBt/event_purity‚0 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.23761416971683502 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9762664437294006 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5139870643615723 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_5_of_6‚1 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.48628416657447815 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9989227056503296 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_7‚1 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9733560085296631 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1401919275522232 % _timestamp‚1.7884487534209244e+09 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.49769777059555054 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.1111111119389534 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5404762029647827 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 6validation_average_jet_accuracy‚0.4386219382286072 8"Purity/*FRt0SRqqt3FBt/event_purity‚0.095238097012043 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.43219685554504395 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9841079711914062 particle/accuracy_3_of_4‚1 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 7"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.45333331823349 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.3333333432674408 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.34889769554138184 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4764367938041687 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.03392857313156128 jet/accuracy_4_of_6‚0 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.16082212328910828 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9663195610046387 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 (particle/accuracy‚0.8532832264900208 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.44694989919662476 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 :"Purity/0FRt1SRqqt2FBt/event_purity‚0.03232758492231369 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.11284151673316956 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 7 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.1088889017701149 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 :"Purity/*FRt1SRqqt2FBt/event_purity‚0.01753712259232998 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ;"Purity/1FRt2SRqqt1FBt/event_purity‚0.005747126415371895 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9864341616630554 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.19301800429821014 jet/accuracy_3_of_7‚0 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 * Purity/2FRt1SRqqt2FBt/FRt_purity‚0.375 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 particle/accuracy_1_of_5‚1 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5392454862594604 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9870601892471313 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3300555348396301 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.4285714328289032 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.014455783180892467 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 )loss/val_total_loss‚47.92530752619673 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5125908255577087 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9351851940155029 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.2614831328392029 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4817606210708618 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5045918226242065 8 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.26944446563720703 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2083333283662796 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.23756372928619385 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 ,jet/accuracy_3_of_3‚0.017556123435497284 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 6 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.190476194024086 particle/accuracy_6_of_7‚1 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.2829839289188385 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13002261519432068 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5560035705566406 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9840329885482788 jet/accuracy_1_of_6‚1 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9547412991523743 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.21211333572864532 jet/accuracy_5_of_5‚0 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 trainer/global_step‚4835 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 6 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.903508722782135 +jet/accuracy_3_of_6‚0.13461539149284363 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5112889409065247 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.017148727551102638 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9620689749717712 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9937499761581421 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.31443455815315247 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.16651690006256104 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 8"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.542816162109375 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.11842317134141922 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.5216049551963806 particle/accuracy_2_of_2‚1 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 7 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.3712962865829468 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.20000000298023224 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8703703880310059 ) Purity/4FRt*SRqqt*FBt/FRt_purity‚0.25 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.1666666716337204 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.6458333134651184 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9951472878456116 6 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.980970561504364 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.3035598695278168 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.4868031442165375 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.16982759535312653 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.22893105447292328 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 6 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.188271626830101 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.09155957400798798 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3214285671710968 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.4166666567325592 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.24715103209018707 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 +jet/accuracy_2_of_5‚0.48657187819480896 + Purity/*FRt2SRqqt2FBt/FRt_purity‚0.3125 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 ( Purity/4FRt1SRqqt0FBt/FRt_purity‚0.5 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.18792621791362762 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.24104689061641693 6 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.503559947013855 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9847952723503113 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5449745655059814 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17866156995296478 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 8"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.517241358757019 jet/accuracy_6_of_7‚0 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 *"Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0.5 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9860354661941528 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3299369812011719 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9920779466629028 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.28125 *"Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0.5 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.12168451398611069 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 particle/accuracy_1_of_3‚1 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 jet/accuracy_5_of_6‚0 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4566666781902313 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9160757064819336 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 particle/accuracy_4_of_6‚1 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.4821428656578064 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.4166666567325592 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9577381014823914 7 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.2629310190677643 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1394824981689453 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9937241077423096 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 9"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.3499999940395355 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5525602698326111 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5190427303314209 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,"Purity/1FRt0SRqqt3FBt/event_purity‚0.125 :"Purity/2FRt0SRqqt2FBt/event_purity‚0.00657894741743803 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 *jet/accuracy_1_of_2‚0.7602952122688293 (particle/f_score‚0.38036778569221497 particle/accuracy_5_of_7‚1 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚1 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5176871418952942 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 *jet/accuracy_2_of_6‚0.4423076808452606 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9980742335319519 particle/accuracy_4_of_4‚1 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6656196117401123 7 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.1452656090259552 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6392262578010559 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.07883637398481369 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 9"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.3499999940395355 7 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.1542649120092392 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.4834374785423279 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.5387051105499268 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.12057279795408249 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.4900508224964142 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9961902499198914 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 epoch‚3 + Purity/1FRt0SRqqt3FBt/FRt_purity‚0.5625 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.4601009786128998 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3611111044883728 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.17666198313236237 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 )jet/accuracy_1_of_5‚0.949251115322113 8 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.30000001192092896 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5510075092315674 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 particle/accuracy_3_of_3‚1 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 jet/accuracy_6_of_6‚0 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9857142567634583 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 8"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.524670422077179 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4423922896385193 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 _step‚395 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.6458333134651184 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5392454862594604 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9285715222358704 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9682528972625732 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.26179298758506775 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5308871269226074 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.1111111119389534 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9902321696281433 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.3659990429878235 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ,&Purity/2FRt3SRqqtB"[eÔ3FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 * Purity/*FRt3SRqqt1FBt/FRt_purity‚0.375 ("Purity/*FRt*SRqqt4FBt/event_purity‚0 particle/accuracy_7_of_7‚0.5 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.18737374246120453 ("Purity/0FRt*SRqqt4FBt/event_purity‚0 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5186278223991394 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.11663108319044113 <"Purity/2FRt*SRqqt2FBt/event_purity‚0.0058139534667134285 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5196197628974915 +jet/accuracy_1_of_1‚0.49639153480529785 +particle/specificity‚0.9577512741088867 ("Purity/0FRt3SRqqt0FBt/event_purity‚0]Á¡Ò|šצ 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.1981857717037201 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.4799640476703644 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.06841396540403366 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9694769978523254 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9907218813896179 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.32567256689071655 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.48848021030426025 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.08953871577978134 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9848309755325317 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.2902558147907257 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4148467481136322 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.006998387165367603 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.3229166567325592 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 jet/accuracy_4_of_7‚0 particle/accuracy_4_of_7‚1 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.23062168061733246 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9886022806167603 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.14837120473384857 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 particle/accuracy_1_of_4‚1 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13488417863845825 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.3580247163772583 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.12574775516986847 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8688279986381531 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9270833134651184 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5385416746139526 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13656951487064362 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ( Purity/4FRt1SRqqt*FBt/FRt_purity‚0.5 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.02421383559703827 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.1496952325105667 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.6969696879386902 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5179523229598999 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 particle/accuracy_1_of_6‚1 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.4838888645172119 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 /"Purity/0FRt0SRqqt3FBt/event_purity‚0.078125 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 jet/accuracy_1_of_7‚1 particle/accuracy_3_of_7‚1 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5833333134651184 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0020674937404692173 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5141317844390869 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 :"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.49294808506965637 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4242689907550812 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 _runtime‚3868.894628462 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9595686197280884 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.0797727033495903 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0037707581650465727 -jet/accuracy_4_of_4‚0.0038580247201025486 particle/accuracy_1_of_1‚1 particle/accuracy_2_of_4‚1 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.019674401730298996 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5585334897041321 particle/accuracy_6_of_6‚1 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.43758171796798706 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3333333432674408 jet/accuracy_7_of_7‚0 +"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.75 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.1111111119389534 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.4234495759010315 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.36369088292121887 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14552029967308044 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9659090638160706 7 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.8571428656578064 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.15698209404945374 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9761709570884705 9"Purity/1FRt1SRqqt0FBt/event_purity‚0.0141415074467659 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.33124998211860657 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5565476417541504 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 ("Purity/*FRt0SRqqt4FBt/event_purity‚0 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.23761416971683502 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9762664437294006 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5139870643615723 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_5_of_6‚1 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.48628416657447815 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9989227056503296 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_7‚1 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9733560085296631 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1401919275522232 % _timestamp‚1.7884487534209244e+09 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.49769777059555054 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.1111111119389534 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5404762029647827 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 6validation_average_jet_accuracy‚0.4386219382286072 8"Purity/*FRt0SRqqt3FBt/event_purity‚0.095238097012043 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.43219685554504395 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9841079711914062 particle/accuracy_3_of_4‚1 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 7"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.45333331823349 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.3333333432674408 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.34889769554138184 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4764367938041687 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.03392857313156128 jet/accuracy_4_of_6‚0 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.16082212328910828 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9663195610046387 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 (particle/accuracy‚0.8532832264900208 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.44694989919662476 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 :"Purity/0FRt1SRqqt2FBt/event_purity‚0.03232758492231369 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.11284151673316956 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 7 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.1088889017701149 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 :"Purity/*FRt1SRqqt2FBt/event_purity‚0.01753712259232998 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ;"Purity/1FRt2SRqqt1FBt/event_purity‚0.005747126415371895 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9864341616630554 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.19301800429821014 jet/accuracy_3_of_7‚0 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 * Purity/2FRt1SRqqt2FBt/FRt_purity‚0.375 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 particle/accuracy_1_of_5‚1 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5392454862594604 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9870601892471313 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3300555348396301 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.4285714328289032 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.014455783180892467 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 )loss/val_total_loss‚47.92530752619673 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5125908255577087 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9351851940155029 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.2614831328392029 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4817606210708618 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5045918226242065 8 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.26944446563720703 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2083333283662796 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.23756372928619385 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 ,jet/accuracy_3_of_3‚0.017556123435497284 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 6 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.190476194024086 particle/accuracy_6_of_7‚1 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.2829839289188385 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13002261519432068 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5560035705566406 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9840329885482788 jet/accuracy_1_of_6‚1 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9547412991523743 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.21211333572864532 jet/accuracy_5_of_5‚0 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 trainer/global_step‚4835 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 6 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.903508722782135 +jet/accuracy_3_of_6‚0.13461539149284363 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5112889409065247 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.017148727551102638 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9620689749717712 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9937499761581421 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.31443455815315247 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.16651690006256104 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 8"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.542816162109375 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.11842317134141922 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.5216049551963806 particle/accuracy_2_of_2‚1 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 7 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.3712962865829468 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.20000000298023224 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8703703880310059 ) Purity/4FRt*SRqqt*FBt/FRt_purity‚0.25 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.1666666716337204 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.6458333134651184 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9951472878456116 6 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.980970561504364 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.3035598695278168 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.4868031442165375 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.16982759535312653 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.22893105447292328 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 6 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.188271626830101 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.09155957400798798 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3214285671710968 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.4166666567325592 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.24715103209018707 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 +jet/accuracy_2_of_5‚0.48657187819480896 + Purity/*FRt2SRqqt2FBt/FRt_purity‚0.3125 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 ( Purity/4FRt1SRqqt0FBt/FRt_purity‚0.5 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.18792621791362762 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.24104689061641693 6 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.503559947013855 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9847952723503113 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5449745655059814 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17866156995296478 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 8"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.517241358757019 jet/accuracy_6_of_7‚0 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 *"Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0.5 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9860354661941528 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3299369812011719 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9920779466629028 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.28125 *"Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0.5 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.12168451398611069 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 particle/accuracy_1_of_3‚1 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 jet/accuracy_5_of_6‚0 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4566666781902313 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9160757064819336 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 particle/accuracy_4_of_6‚1 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.4821428656578064 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.4166666567325592 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9577381014823914 7 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.2629310190677643 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1394824981689453 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9937241077423096 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 9"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.3499999940395355 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5525602698326111 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5190427303314209 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,"Purity/1FRt0SRqqt3FBt/event_purity‚0.125 :"Purity/2FRt0SRqqt2FBt/event_purity‚0.00657894741743803 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 *jet/accuracy_1_of_2‚0.7602952122688293 (particle/f_score‚0.38036778569221497 particle/accuracy_5_of_7‚1 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚1 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5176871418952942 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 *jet/accuracy_2_of_6‚0.4423076808452606 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9980742335319519 particle/accuracy_4_of_4‚1 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6656196117401123 7 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.1452656090259552 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6392262578010559 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.07883637398481369 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 9"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.3499999940395355 7 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.1542649120092392 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.4834374785423279 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.5387051105499268 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.12057279795408249 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.4900508224964142 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9961902499198914 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 epoch‚3 + Purity/1FRt0SRqqt3FBt/FRt_purity‚0.5625 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.4601009786128998 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3611111044883728 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.17666198313236237 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 )jet/accuracy_1_of_5‚0.949251115322113 8 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.30000001192092896 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5510075092315674 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 particle/accuracy_3_of_3‚1 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 jet/accuracy_6_of_6‚0 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9857142567634583 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 8"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.524670422077179 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4423922896385193 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 _step‚395 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.6458333134651184 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5392454862594604 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9285715222358704 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9682528972625732 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.26179298758506775 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5308871269226074 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.1111111119389534 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9902321696281433 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.3659990429878235 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 * Purity/*FRt3SRqqt1FBt/FRt_purity‚0.375 ("Purity/*FRt*SRqqt4FBt/event_purity‚0 particle/accuracy_7_of_7‚0.5 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.18737374246120453 ("Purity/0FRt*SRqqt4FBt/event_purity‚0 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5186278223991394 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.11663108319044113 <"Purity/2FRt*SRqqt2FBt/event_purity‚0.0058139534667134285 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5196197628974915 +jet/accuracy_1_of_1‚0.49639153480529785 +particle/specificity‚0.9577512741088867 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5301687121391296 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3392690420150757 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.18827159702777863 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5900925993919373 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5271347761154175 +jet/accuracy_2_of_3‚0.24027033150196075 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.03394412249326706 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9877298474311829 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.43969300389289856 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.4343397319316864 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.06594319641590118 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 ,particle/sensitivity‚0.28862056136131287 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5526315569877625 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.4354199767112732 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.1603776216506958 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.9333333373069763 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.2020781934261322 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_4‚0.9013838171958923 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5594624876976013 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.03186967596411705 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0014430014416575432 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.014852607622742653 particle/accuracy_5_of_5‚1 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.05263157933950424 ("Purity/0FRt0SRqqt4FBt/event_purity‚0 ("Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5463369488716125 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5457584261894226 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 jet/accuracy_2_of_7‚0.5 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.1907934546470642 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9769506454467773 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.10111435502767563 particle/accuracy_1_of_2‚1 jet/accuracy_5_of_7‚0 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.06382978707551956 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5380474925041199 9"Purity/3FRt1SRqqtn‰@*FBt/SRqqt_purity‚0.5023148059844971 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9464285969734192 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.4814814329147339 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5484960675239563 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ,jet/accuracy_3_of_4‚0.047013767063617706 particle/accuracy_2_of_5‚1 particle/accuracy_4_of_5‚1 :"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.44999998807907104 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.39417287707328796 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 +jet/accuracy_3_of_5‚0.08395417034626007 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9851285219192505 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.5642702579498291 9"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.4426426291465759 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 & Purity/*FRt1SRqqt4FBt/FRt_purity‚1 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.038266781717538834 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.48891738057136536 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.011798255145549774 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.1740759313106537 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9433333873748779 ) Purity/4FRt*SRqqt0FBt/FRt_purity‚0.25 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5124761462211609 + Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9375 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4583333730697632 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3819444477558136 particle/accuracy_3_of_6‚1 +"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.75 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.09645634144544601 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ("Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0 particle/accuracy_2_of_6‚1 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9980742335319519 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 6 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.991583526134491 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3673110902309418 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.15526314079761505 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.17052695155143738 7 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.1982758641242981 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1191326379776001 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5614035129547119 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5624845027923584 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5535714626312256 6 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.995573878288269 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 :"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.43947139382362366 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.14837120473384857 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.3362351059913635 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 , Purity/1FRt3SRqqt1FBt/FRt_purity‚0.40625 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 particle/accuracy_2_of_7‚1 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8140152096748352 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.15909090638160706 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 6 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.529411792755127 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.21938589215278625 )jet/accuracy_2_of_4‚0.337873637676239 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.20000000298023224 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.1666666716337204 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.17830407619476318 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.6666666269302368 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 -jet/accuracy_4_of_5‚0.0049923197366297245‹ŒšI’›‚(:‡ òŸæÔ¿¾ÿü gpu.0.gpu‚6.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚34.0 gpu.0.powerWatts‚122.822' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.5644 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0n~ÉZœ‚(:Ï òŸæÔ²•îÿ disk.drbd2080.out‚0 disk.sda3.in‚0$ memory_percent‚97.78268337249756 cpu‚25.367083473082754 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda.in‚0 disk.sda1.in‚0* disk./.usagePercent‚20.177281428549897 disk.drbd2266.out‚0 disk.drbd2080.in‚0# proc.memory.rssMB‚ 2854.14453125 disk.sda1.out‚0 disk.dm-1.out‚ 148.72265625 disk.drbd1716.out‚0 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚14892.8974609375 disk.sda3.out‚ 148.72265625$ disk./.usageGB‚300.6560745239258 disk.sda2.in‚0 disk.dm-1.in‚0 disk.drbd2265.out‚0( proc.memory.availableMB‚ 363.28515625* proc.memory.percent‚17.420315742492676 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚14892.8974609375 network.sent‚ 1.0997754e+07 network.recv‚ 1.42593e+06 disk.dm-0.in‚0.078125 disk.drbd1716.in‚0 proc.cpu.threads‚17 disk.rbd0.out‚0 disk.nvme0c0n1.in‚0.078125" disk.dm-0.out‚14892.8974609375 disk.sda.out‚ 148.72265625 disk.rbd1.in‚0 disk.rbd1.out‚ 19.9697265625Xrpq2- _wandbruntime‚3883  _runtime‚3883è8“ûÇžÁ % _timestamp‚1.7884487676185138e+09 _runtime‚3883.090215458 _step‚396 & lr-AdamW/pg1‚9.480814872292184e-05 & lr-AdamW/pg2‚9.480814872292184e-05 trainer/global_step‚4849s6¢3ÌŸÆ _step‚396 & lr-AdamW/pg1‚9.480814872292184e-05 & lr-AdamW/pg2‚9.480814872292184e-05 trainer/global_step‚4849 % _timestamp‚1.7884487676185138e+09 _runtime‚3883.090215458Œä}2 - _wandbruntime‚3883  _runtime‚3883ä‹œÝ'¡×O ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4068108491776 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4068108491776 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚312837429079552 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚91689348 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚286678426648064 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚282610300153344 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚275194743995392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23296778 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41259508 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884487676193354e+09 _runtime‚3883.091157446 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 trainer/global_step‚4849 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 162397658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚279247889303760 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4067452129280 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚91669310 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4052473831252 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚286677752282624 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4067452129280 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4266035194880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚275194761034752 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41259510 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 162377618 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70708308 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4266198818816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚279247217826644 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _step‚397 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚286678426648064 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚286677752282624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚70708310 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4053128269008 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚91669310 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚308571417824768RÈ]_â'¢ÜO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚286677752282624 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4067452129280 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4266035194880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚275194761034752 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41259510 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 162377618 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚70708308 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4266198818816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚279247217826644 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _step‚397 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚286678426648064 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚286677752282624 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚70708310 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4053128269008 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚91669310 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚308571417824768 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4068108491776 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4068108491776 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚312837429079552 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚91689348 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚286678426648064 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚282610300153344 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚275194743995392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23296778 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚41259508 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884487676193354e+09 _runtime‚3883.091157446 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 trainer/global_step‚4849 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 162397658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚279247889303760 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4067452129280 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚91669310 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4052473831252 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038h“72£- _wandbruntime‚3883  _runtime‚3883E€P?Ÿ¤áM RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 162430154 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4266894360064 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚308633068562432 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚275249665165312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4068927932928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚312900126533120 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚41267676 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚282666671195136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4068271557120 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4068271557120 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚308633044622336 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41267678 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚279303609869492 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _step‚398 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚286735599128064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚31ÇúH2899938982400 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚91707828 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23301277 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚91687764 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4053944704180 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚282666653192192 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚282666671195136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚64568953 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4267057970688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4053290266216 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚91707828 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884487683202105e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 162430154 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚3883.792350776 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 162410088 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4068927932928 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚91687764 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23301717 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4068271557120 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚308633044622336 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚41267678 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚279303609869492 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _step‚398 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚286735599128064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚312899938982400 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚91707828 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23301277 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚91687764 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4053944704180 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚282666653192192 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚282666671195136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚64568953 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4267057970688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4053290266216 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚91707828 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884487683202105e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 162430154 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚3883.792350776 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 162410088 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4068927932928 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚91687764 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23301717 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚308633068562432 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚275249665165312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4068927932928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚312900126533120 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430Žô ¿è2¦- _wandbruntime‚3883  _runtime‚3883&“ç[§Õ 1loss/SRqqt3/detection_loss‚1.1764856225022904 .loss/FBt1/detection_loss‚2.399101823568344 2loss/SRqqt2/assignment_loss‚1.8593731654702492 /loss/FRt1/detection_loss‚2.9238688308617165 /loss/FRt4/detection_loss‚2.2440837042575534 % _timestamp‚1.7884487683219733e+09 1loss/SRqqt4/assignment_loss‚1.414750713396531 0loss/FBt1/assignment_loss‚0.5285388114078842 0loss/FBt3/assignment_loss‚0.2707633326592227 1loss/SRqqt2/detection_loss‚1.7362204253111246 trainer/global_step‚4849 /loss/FBt2/detection_loss‚1.9773332496484122 0loss/FRt4/assignment_loss‚2.9429188045588406 /loss/FRt3/detection_loss‚2.1433743202970144 1loss/SRqqt4/detection_loss‚1.4536367652507929 _runtime‚ 3883.79350316 0loss/FRt2/assignment_loss‚3.9669146066488223 1loss/SRqqt3/assignment_loss‚1.307792866917757 /loss/FRt2/detection_loss‚2.3647199584994207 &loss/total_loss‚45.672256369765094 /loss/FRt1/assignment_loss‚3.665285139248289 1loss/SRqqt1/assignment_loss‚2.568755410868546 1loss/FBt4/assignment_loss‚0.12443800472328803 epoch‚4 0loss/FRt3/assignment_loss‚1.8359946622734977 0loss/FBt2/assignment_loss‚0.5097559854331848 1loss/SRqqt1/detection_loss‚2.4495670466587462 /loss/FBt3/detection_loss‚0.7102626053398693 /loss/FBt4/detection_loss‚3.0983205139636993 _step‚399Z„II`¨Ú /loss/FRt4/detection_loss‚2.2440837042575534 % _timestamp‚1.7884487683219733e+09 1loss/SRqqt4/assignment_loss‚1.414750713396531 0loss/FBt1/assignment_loss‚0.5285388114078842 0loss/FBt3/assignment_loss‚0.2707633326592227 1loss/SRqqt2/detection_loss‚1.7362204253111246 trainer/global_step‚4849 /loss/FBt2/detection_loss‚1.9773332496484122 0loss/FRt4/assignment_loss‚2.9429188045588406 /loss/FRt3/detection_loss‚2.1433743202970144 1loss/SRqqt4/detection_loss‚1.4536367652507929 _runtime‚ 3883.79350316 0loss/FRt2/assignment_loss‚3.9669146066488223 1loss/SRqqt3/assignment_loss‚1.307792866917757 /loss/FRt2/detection_loss‚2.3647199584994207 &loss/total_loss‚45.672256369765094 /loss/FRt1/assignment_loss‚3.665285139248289 1loss/SRqqt1/assignment_loss‚2.568755410868546 1loss/FBt4/assignment_loss‚0.12443800472328803 epoch‚4 0loss/FRt3/assignment_loss‚1.8359946622734977 0loss/FBt2/assignment_loss‚0.5097559854331848 1loss/SRqqt1/detection_loss‚2.4495670466587462 /loss/FBt3/detection_loss‚0.7102626053398693 /loss/FBt4/detection_loss‚3.0983205139636993 _step‚399 1loss/SRqqt3/detection_loss‚1.1764856225022904 .loss/FBt1/detection_loss‚2.399101823568344 2loss/SRqqt2/assignment_loss‚1.8593731654702492 /loss/FRt1/detection_loss‚2.9238688308617165ƒc “©‚(:ˆ  æÔÕÇûü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚176.142' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.2284 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0<.ºÃSª‚(:È  æÔÉ€†ÿ disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0# memory_percent‚94.1901445388794 network.sent‚ 1.1142828e+07 disk.sda3.in‚0 disk.rbd0.in‚0& disk.nvme0n1.out‚15902.48486328125 disk.dm-1.out‚ 149.15625 proc.cpu.threads‚17 disk.rbd1.out‚ 20.0322265625 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0# proc.memory.rssMB‚ 2644.20703125 cpu‚25.364090892987726 network.recv‚ 1.432053e+06 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.dm-1.in‚0( proc.memory.availableMB‚ 951.88671875) proc.memory.percent‚16.13895893096924 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 149.15625 disk.drbd2080.in‚0* disk./.usagePercent‚20.178483383923155 disk.sda.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0$ disk./.usageGB‚300.6739845275879( disk.nvme0c0n1.out‚15902.48486328125 disk.rbd0.out‚0# disk.dm-0.out‚15902.48486328125 disk.sda.out‚ 149.15625 disk.sda1.out‚0îá|ˆ“«‚(:ˆ  æÔÚ°ˆý gpu.0.gpu‚50.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚183.297' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.6594 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÙwÞ3U¬‚(:Ê  æÔ¾ó‹ÿ disk.rbd1.out‚ 20.0322265625 disk.dm-1.in‚0 disk.sda2.out‚0 cpu‚25.362891218661495 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125* disk./.usagePercent‚20.179514838278607$ disk.nvme0n1.out‚15950.423828125& disk.nvme0c0n1.out‚15950.423828125 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.sda2.in‚0( proc.memory.availableMB‚ 952.11328125 network.sent‚ 1.1251594e+07 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 proc.cpu.threads‚17 disk.sda3.out‚ 149.88671875 disk.drbd2266.out‚0 disk.sda.in‚0 disk.dm-1.out‚ 149.88671875 disk.sda.out‚ 149.88671875$ disk./.usageGB‚300.6893539428711 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.sda3.in‚0 network.recv‚ 1.43908e+06 disk.drbd2265.in‚0 disk.rbd0.out‚0! disk.dm-0.out‚15950.423828125 disk.sda1.in‚0# memory_percent‚94.1887617111206# proc.memory.rssMB‚ 2644.20703125) proc.memory.percent‚16.13895893096924iùôž­‚(:“ Ÿ æԀÄý gpu.0.gpu‚39.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚124.866' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.973200000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0kªØºW®‚(:Ì Ÿ æÔªš˜€" disk.dm-0.out‚15995.2099609375 disk.sda1.out‚0 proc.cpu.threads‚17 network.recv‚ 1.442437e+06% disk.nvme0n1.out‚15995.2099609375$ memory_percent‚94.18878555297852) proc.memory.percent‚16.13895893096924 cpu‚25.361686701582762' disk.nvme0c0n1.out‚15995.2099609375 disk.drbd2266.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 20.322265625 disk.nvme0c0n1.in‚0.078125 disk.dm-1.in‚0* disk./.usagePercent‚20.180491763029266 disk.sda3.in‚0 disk.sda3.out‚ 150.38671875 disk.dm-0.in‚0.078125 disk.sda.out‚ 150.38671875 disk.rbd0.in‚0 disk.sda.in‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd1716.out‚0& proc.memory.availableMB‚ 952.109375 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.dm-1.out‚ 150.38671875 disk.drbd1716.in‚0# proc.memory.rssMB‚ 2644.20703125 network.sent‚ 1.1256749e+07$ disk./.usageGB‚300.7039108276367ÿº“Ý2¯- _wandbruntime‚3920  _runtime‚3920ÈhåÅ°¿ % lr-AdamW/pg1‚9.40112708236765e-05 % lr-AdamW/pg2‚9.40112708236765e-05 trainer/global_step‚4899 % _timestamp‚1.7884488046148312e+09 _runtime‚3920.086464587 _step‚400ñ– 汀 % _timestamp‚1.7884488046148312e+09 _runtime‚3920.086464587 _step‚400 % lr-AdamW/pg1‚9.40112708236765e-05 % lr-AdamW/pg2‚9.40112708236765e-05 trainer/global_step‚4899’`;2²- _wandbruntime‚3920  _runtime‚3920Ò5æ¹#³ÖO L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚92595362 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚65191030 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚285427952095744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚4899 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 $ _timestamp‚1.788448804615579e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71409108 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚289536390030848 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚282033259694052 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚289537064396288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71409108 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4093963608416 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 _runtime‚3920.087445799 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 164004470 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23523122 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23523549 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚315963067368960 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚92595362 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚311653930767872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚282033931171168 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4309136601088 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚277939967562752 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚92615400 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4109094297600 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 X$ÅmÆ#KDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41667910 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚315963254932992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4109094297600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚289536390030848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚92615400ä™Óá'´ÛO N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 _runtime‚3920.087445799 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 164004470 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23523122 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23523549 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚315963067368960 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚92595362 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚311653930767872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚282033931171168 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4309136601088 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚277939967562752 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚92615400 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4109094297600 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚41667910 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚315963254932992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4109094297600 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚289536390030848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚92615400 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚92595362 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚65191030 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚285427952095744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚4899 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚427 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 $ _timestamp‚1.788448804615579e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚71409108 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚289536390030848 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚282033259694052 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚289537064396288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚71409108 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚429 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4093963608416 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944‘ûS !2µ- _wandbruntime‚3920  _runtime‚3920Û±–éç&¶áM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4109257362944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 164036940 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4309992630784 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _step‚402 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4109913738752 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚92633880 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚282088980259576 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚311715605445632 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚92633880 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71423124 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚311715581505536 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4094125605624 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚65204058 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚437 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚277994871693312 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71423126 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚289594236876288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚285484323137536 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4310158338560 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4109913738752 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23527980 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚3920.772768028 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 164036940 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚285484305134592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23527545 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71423124 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚277994854653952 % _timestamp‚1.7884488053005934e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚92613816 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚92613816 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 164057006 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚289594236876288 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65203621 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚435 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚316025763784192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚285484323137536 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430™ ˆì&·æM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4109913738752 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23527980 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚3920.772768028 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 164036940 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚285484305134592 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚23527545 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚71423124 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚277994854653952 % _timestamp‚1.7884488053005934e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚92613816 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚92613816 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 164057006 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚289594236876288 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65203621 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚435 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚316025763784192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚285484323137536 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4109257362944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 164036940 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4309992630784 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _step‚402 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4109913738752 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚92633880 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚282088980259576 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚311715605445632 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚92633880 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚71423124 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚311715581505536 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4094125605624 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚65204058 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚437 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚277994871693312 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚71423126 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚289594236876288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚285484323137536 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4310158338560 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025’y§F_2¸- _wandbruntime‚3920  _runtime‚3920±=›[¹Õ 1loss/SRqqt2/detection_loss‚1.8032973878913456 1loss/FBt4/assignment_loss‚0.32275733792084793 0loss/FRt4/assignment_loss‚3.2518924607170954 1loss/SRqqt4/detection_loss‚1.5618797312180202 .loss/FBt2/detection_loss‚2.327978120318481 epoch‚4 _step‚403 2loss/SRqqt2/assignment_loss‚1.4475987320144972 0loss/SRqqt3/detection_loss‚1.384608854430125 _runtime‚3920.773794225 &loss/total_loss‚46.101893598387264 0loss/FRt2/assignment_loss‚3.8693078603855398 0loss/FBt1/assignment_loss‚0.3249170732487156 .loss/FRt2/detection_loss‚2.064703891443652 /loss/FRt3/detection_loss‚1.8942807022304762 0loss/FBt2/assignment_loss‚0.4671825401926201 .loss/FBt1/detection_loss‚2.781152215870944 /loss/FBt3/detection_loss‚0.8925968121215136 % _timestamp‚1.7884488053022802e+09 /loss/FRt1/assignment_loss‚4.677100465847896 /loss/FRt1/detection_loss‚3.2807732614187093 0loss/SRqqt1/detection_loss‚2.689815346683775 0loss/FRt3/assignment_loss‚1.8485268269266402 1loss/SRqqt1/assignment_loss‚1.947344574545111 2loss/SRqqt4/assignment_loss‚1.3580367057273786 trainer/global_step‚4899 /loss/FBt4/detection_loss‚2.3589185774326324 2loss/SRqqt3/assignment_loss‚1.0910274657585166 1loss/FBt3/assignment_loss‚0.22948410210898837 /loss/FRt4/detection_loss‚2.2267125519337476l•ÜÝ2ºÚ 2loss/SRqqt4/assignment_loss‚1.35803670Ú—ý‡.57273786 trainer/global_step‚4899 /loss/FBt4/detection_loss‚2.3589185774326324 2loss/SRqqt3/assignment_loss‚1.0910274657585166 1loss/FBt3/assignment_loss‚0.22948410210898837 /loss/FRt4/detection_loss‚2.2267125519337476 1loss/SRqqt2/detection_loss‚1.8032973878913456 1loss/FBt4/assignment_loss‚0.32275733792084793 0loss/FRt4/assignment_loss‚3.2518924607170954 1loss/SRqqt4/detection_loss‚1.5618797312180202 .loss/FBt2/detection_loss‚2.327978120318481 epoch‚4 _step‚403 2loss/SRqqt2/assignment_loss‚1.4475987320144972 0loss/SRqqt3/detection_loss‚1.384608854430125 _runtime‚3920.773794225 &loss/total_loss‚46.101893598387264 0loss/FRt2/assignment_loss‚3.8693078603855398 0loss/FBt1/assignment_loss‚0.3249170732487156 .loss/FRt2/detection_loss‚2.064703891443652 /loss/FRt3/detection_loss‚1.8942807022304762 0loss/FBt2/assignment_loss‚0.4671825401926201 .loss/FBt1/detection_loss‚2.781152215870944 /loss/FBt3/detection_loss‚0.8925968121215136 % _timestamp‚1.7884488053022802e+09 /loss/FRt1/assignment_loss‚4.677100465847896 /loss/FRt1/detection_loss‚3.2807732614187093 0loss/SRqqt1/detection_loss‚2.689815346683775 0loss/FRt3/assignment_loss‚1.8485268269266402 1loss/SRqqt1/assignment_loss‚1.947344574545111“º%6•»‚(:’ ® æԵݠý gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚171.372' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚34.27440000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÃjÑY¼‚(:Î ® æÔÇô¨ÿ# proc.memory.rssMB‚ 2644.20703125) proc.memory.percent‚16.13895893096924 disk.sda2.out‚0 disk.dm-1.in‚0 disk.drbd2080.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 150.8828125 disk.drbd1716.in‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.sda.in‚0 cpu‚25.360454153186595& disk.nvme0n1.out‚16058.84912109375) disk./.usagePercent‚20.18163099655664 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0$ memory_percent‚94.18919086456299 network.sent‚ 1.1263421e+07% disk./.usageGB‚300.72088623046875 disk.nvme0c0n1.in‚0.078125 network.recv‚ 1.447458e+06( disk.nvme0c0n1.out‚16058.84912109375 disk.rbd0.out‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚16058.84912109375 disk.sda1.in‚0 disk.rbd1.out‚ 20.322265625( proc.memory.availableMB‚ 952.04296875 disk.sda3.out‚ 150.8828125 disk.rbd0.in‚0 disk.sda.out‚ 150.8828125 disk.sda2.in‚0¸]Žù“½‚(:ˆ ½ æÔšÞöü gpu.0.gpu‚38.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.872' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.1744 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0+X¾‚(:Í ½ æԾЌ€ disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0$ disk./.usageGB‚300.7281951904297 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚16106.91552734375 disk.dm-1.out‚ 151.6015625 disk.drbd2080.out‚0 disk.sda.in‚0 disk.sda.out‚ 151.6015625 disk.sda2.out‚0( disk.nvme0c0n1.out‚16106.91552734375 disk.sda3.out‚ 151.6015625 disk.drbd2266.in‚0( proc.memory.availableMB‚ 952.04296875# proc.memory.rssMB‚ 2644.20703125 cpu‚25.35927222230405 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0# disk.dm-0.out‚16106.91552734375) proc.memory.percent‚16.13895893096924 network.recv‚ 1.452886e+06 disk.sda3.in‚0 disk.rbd0.in‚0$ memory_percent‚94.18919086456299 network.sent‚ 1.1371147e+07 disk.rbd1.out‚ 20.470703125 disk.dm-1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda1.in‚0* disk./.usagePercent‚20.182121506992242 proc.cpu.threads‚17ýNÊ2¿- _wandbruntime‚3956  _runtime‚3956ÓP”þÇÀÁ _runtime‚3956.813778753 _step‚404 & lr-AdamW/pg1‚9.320800167074349e-05 & lr-AdamW/pg2‚9.320800167074349e-05 trainer/global_step‚4949 % _timestamp‚1.7884488413421829e+09”9ŒØÌÁÆ & lr-AdamW/pg1‚9.320800167074349e-05 & lr-AdamW/pg2‚9.320800167074349e-05 trainer/global_step‚4949 % _timestamp‚1.7884488413421829e+09 _runtime‚3956.813778753 _step‚404”Ž£[ 2Â- _wandbruntime‚3956  _runtime‚3956:—ˆÝ'Ã×O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42076308 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚292395702144512 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚280685157051392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚288245622041088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚314736491591168 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4352217192960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚288245622041088 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884488413430598e+09 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 165631322 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚93521414 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚319088866116608 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4134144510068 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23750447 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚319088684844032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4134798947824 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚93521414 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 _step‚405 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72109910 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72109908 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23750882 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4352374525440 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4149423740928 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 165651362 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚284819301561460 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚93541452 _runtime‚3956.814977114 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚292395702144512 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4149423740928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚65827192 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚65826755 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚435 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72109910 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚288245604038144 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚280685174090752 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚284819973038576 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚314736467651072 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42076310 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚292395027779072 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚292395027779072 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4134798947824 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚93521414 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 _step‚405 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72109910 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72109908 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23750882 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4352374525440 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4149423740928 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 165651362 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚284819301561460 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚93541452 _runtime‚3956.814977114 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚292395702144512 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4149423740928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚65827192 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚65826755 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚435 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72109910 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚288245604038144 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚280685174090752 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚284819973038576 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚314736467651072•¥iQ2Å- _wandbruntime‚3957  _runtime‚3957JSº®ÆáM P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4150899544576 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚288301957076992 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚280740078221312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚288301957076992 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4150899544576 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚319151187454976 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4135615382996 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚292452200245760 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 165663792 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 165683858 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4150243168768 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72123926 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚314798118388736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚280740061181952 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65839489 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚319151379200000 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72123924 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚288301975079936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚93559932 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚284875022126984 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OBDev›è¨Þ9 iceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚65839928 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _step‚406 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚292452874624512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚284875693604308 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4134960945032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42084476 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚314798142328832 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4353069066240 % _timestamp‚1.7884488420298412e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚93559932 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚292452874624512 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 165683858 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 165663792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4150243168768 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4353236871168 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0¥v¿ì&ÇæM H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42084478 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚288301975079936 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚65839489 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚319151379200000 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72123924 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚288301975079936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚93559932 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚284875022126984 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚65839928 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _step‚406 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚292452874624512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚284875693604308 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4134960945032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42084476 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚314798142328832 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4353069066240 % _timestamp‚1.7884488420298412e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚93559932 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚292452874624512 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 165683858 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 165663792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4150243168768 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4353236871168 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4150899544576 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚288301957076992 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚280740078221312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚288301957076992 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4150899544576 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚319151187454976 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4135615382996 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚292452200245760 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 165663792 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 165683858 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4150243168768 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72123926 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚314798118388736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚280740061181952 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4395550673408 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4395391243776 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72810708 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72810708 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23979068 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚291063255980544 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42484710 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚283430363579392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚291063255980544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 167258174 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚94447466 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚295253665527296 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚295254339892736 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚94447466 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4175634287232 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4191065909248 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚317819004534272 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚283430380618752 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚66463329 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚322214579147776 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚94467504 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚287606014905984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚291063273983488 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23978621 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1ëŒZNñÖÛO JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚449 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 % _timestamp‚1.7884488784173365e+09 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚94467504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚72810710 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚295254339892736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42484708 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚ 3993.88921459 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚317819028474368 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4174979849476 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4190409546752 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 41904095467S¹x|ð52 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚409 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚72810710 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚4999 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚295253665527296 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4395550673408 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4395391243776 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚72810708 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚72810708 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚23979068 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚291063255980544 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42484710 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚283430363579392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚291063255980544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 167258174 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚94447466 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚295253665527296 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚295254339892736 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚94447466 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4175634287232 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4191065909248 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚317819004534272 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚283430380618752 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚66463329 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚322214579147776 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚94467504 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚287606014905984 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚291063273983488 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚23978621 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4191065909248 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096™³1õ2×- _wandbruntime‚3994  _runtime‚3994Õ.Àæ&ØàM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚291119627022336 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚66475945 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚457 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72824726 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚4999 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚295311512372736 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72824726 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚295310837993984 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4191228974592 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23983526 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚94485984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚295311512372736 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 167310710 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚410 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 167310710 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72824724 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚66476404 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚287661735471716 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚287661063994392 _runtime‚ 3994.56713514 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 167290644 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚291119609019392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4191885350400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72824724 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42492878 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42492876 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚283485284749312 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚317880655271936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚283485267709952 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4176450722404 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884488790948544e+09 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚94485984 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚291119609019392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚322277096010752 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚94485984 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚291119609019392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚322277096010752 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚66475945 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚457 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚72824726 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚4999 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚295311512372736 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚72824726 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚295310837993984 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4191228974592 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚23983526 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚94485984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚295311512372736 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 167310710 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚410 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 167310710 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚72824724 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚66476404 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚287661735471716 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚287661063994392 _runtime‚ 3994.56713514 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 167290644 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚291119609019392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4191885350400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚72824724 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42492878 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42492876 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚283485284749312 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚317880655271936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚283485267709952 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4176450722404 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884488790948544e+09šŒG}‹2Ú- _wandbruntime‚3994  _runtime‚3994¾ŠXÛÒ /loss/FRt1/assignment_loss‚4.877582443171534 1loss/SRqqt3/assignment_loss‚0.939687875730376 2loss/SRqqt4/assignment_loss‚1.6663221677144369 0loss/FBt2/assignment_loss‚0.5412316332256221 .loss/FRt1/detection_loss‚2.888735224460733 .loss/FRt2/detection_loss‚2.237512942403555 % _timestamp‚1.7884488790965743e+09 /loss/FRt3/detection_loss‚1.9026409685611725 1loss/SRqqt3/detection_loss‚1.0095293411924955 /loss/FBt3/detection_loss‚0.8308046216813799 0loss/FBt4/assignment_loss‚0.6798177808523178 1loss/SRqqt4/detection_loss‚1.7439281602700551 %loss/total_loss‚48.80397640133546 /loss/FRt4/detection_loss‚2.3319752663373947 1loss/SRqqt1/detection_loss‚2.5566491098964916 /loss/FBt1/detection_loss‚3.0689286962151527 epoch‚4 _runtime‚3994.568057831 0loss/FRt4/assignment_loss‚3.5394588736600654 trainer/global_step‚4999 /loss/FRt2/assignment_loss‚3.455441373089949 0loss/FRt3/assignment_loss‚2.2097382662343046 .loss/FBt4/detection_loss‚4.221372008323669 2loss/SRqqt1/assignment_loss‚1.9756969844593721 /loss/FBt1/assignment_loss‚0.522322145072394 1loss/FBt3/assignment_loss‚0.23069836314875786 1loss/SRqqt2/detection_loss‚1.7336013133034986 .loss/FBt2/detection_loss‚2.062959055105845 2loss/SRqqt2/assignment_loss‚1.5773417872248912 _step‚411Ôšî]Ü× /loss/FRt1/assignment_loss‚4.877582443171534 1loss/SRqqt3/assignment_loss‚0.939687875730376 2loss/SRqqt4/assignment_loss‚1.6663221677144369 0loss/FBt2/assignment_loss‚0.5412316332256221 .loss/FRt1/detection_loss‚2.888735224460733 .loss/FRt2/detection_loss‚2.237512942403555 % _timestamp‚1.7884488790965743e+09 /loss/FRt3/detection_loss‚1.9026409685611725 1loss/SRqqt3/detection_loss‚1.0095293411924955 /loss/FBt3/detection_loss‚0.8308046216813799 0loss/FBt4/assignment_loss‚0.6798177808523178 1loss/SRqqt4/detection_loss‚1.7439281602700551 %loss/total_loss‚48.80397640133546 /loss/FRt4/detection_loss‚2.3319752663373947 1loss/SRqqt1/detection_loss‚2.5566491098964916 /loss/FBt1/detection_loss‚3.0689286962151527 epoch‚4 _runtime‚3994.568057831 0loss/FRt4/assignment_loss‚3.5394588736600654 trainer/global_step‚4999 /loss/FRt2/assignment_loss‚3.455441373089949 0loss/FRt3/assignment_loss‚2.2097382662343046 .loss/FBt4/detection_loss‚4.221372008323669 2loss/SRqqt1/assignment_loss‚1.9756969844593721 /loss/FBt1/assignment_loss‚0.522322145072394 1loss/FBt3/assignment_loss‚0.23069836314875786 1loss/SRqqt2/detection_loss‚1.7336013133034986 .loss/FBt2/detection_loss‚2.062959055105845 2loss/SRqqt2/assignment_loss‚1.5773417872248912 _step‚411›0ÕBžÝ‚(:“ ù æÔÑêý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.123' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.624599999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÙÒìyGÞ‚(:¼ ù æÔ•‘èÿ) proc.memory.percent‚16.13898277282715$ disk.nvme0c0n1.out‚ 16310.1328125 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.dm-1.out‚ 153.828125 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-0.out‚ 16310.1328125 disk.sda2.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 cpu‚25.354676202221675 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.sda3.out‚ 153.828125 disk.sda.in‚0 disk.sda.out‚ 153.828125 disk.sda1.out‚0 disk.drbd1716.in‚0$ memory_percent‚94.19755935668945 network.sent‚ 1.149754e+07 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 network.recv‚ 1.471846e+06* disk./.usagePercent‚20.181722647253896 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0% disk./.usageGB‚300.72225189208984 disk.rbd1.out‚ 20.791015625" disk.nvme0n1.out‚ 16310.1328125& proc.memory.availableMB‚ 950.671875" proc.memory.rssMB‚ 2644.2109375Šv´ûß‚(:ˆ ˆ¡æÔ唡ý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.274' gpu.0.enforcedPowerLimitWatts‚500.0 –ÒÒÕ˜gpu.0.powerPercent‚35.8548 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ûfØ«Uà‚(:Ê ˆ¡æÔ®—ôþ disk.nvme0n1.in‚0.078125 disk.dm-1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 154.4296875 disk.rbd0.out‚0 disk.sda1.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 20.791015625 cpu‚25.353559231601388 disk.sda2.in‚0 network.recv‚ 1.479119e+06 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0" disk.dm-0.out‚16353.4931640625$ memory_percent‚94.19910907745361 proc.cpu.threads‚17$ disk./.usageGB‚300.7448387145996 disk.drbd2265.out‚0 disk.sda.in‚0 disk.sda2.out‚0( proc.memory.availableMB‚ 950.41796875" proc.memory.rssMB‚ 2644.2109375 network.sent‚ 1.1606523e+07 disk.drbd1716.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 154.4296875% disk.nvme0n1.out‚16353.4931640625 disk.drbd1716.out‚0 disk.dm-1.out‚ 154.4296875 disk.drbd2265.in‚0 disk.drbd2080.in‚0) proc.memory.percent‚16.13898277282715* disk./.usagePercent‚20.183238467864133 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚16353.4931640625 disk.sda3.in‚0 disk.sda1.out‚0¹)?*2á- _wandbruntime‚4030  _runtime‚4030¹.ªXÇâÁ _step‚412 & lr-AdamW/pg1‚9.158296050120631e-05 & lr-AdamW/pg2‚9.158296050120631e-05 trainer/global_step‚5049 % _timestamp‚1.7884489149521894e+09 _runtime‚4030.423786121ŒfnÃÌãÆ & lr-AdamW/pg1‚9.158296050120631e-05 & lr-AdamW/pg2‚9.158296050120631e-05 trainer/global_step‚5049 % _timestamp‚1.7884489149521894e+09 _runtime‚4030.423786121 _step‚412œ%°”õ2ä- _wandbruntime‚4030  _runtime‚4030ŒÌ¤Ý'å×O UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚290391385296276 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚67100400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42893108 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚298112303275520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚298112977640960 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 _step‚413 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 168885026 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚325340098780160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4231395352576 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚290392056773392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚95393556 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 168905066 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73511510 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844æ[¸îâ'æÜO ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4216469626640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4438725180928 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚95393556 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚293880925925888 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚286175570107392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4232051715072 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚298112303275520 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚298112977640960 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4231395352576 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚293880907922944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4438557362688 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 % _timestamp‚1.7884489149529364e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚95373518 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚325340290538496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚293880907922944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚320901565357568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚286175587146752 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚42893110 trainer/global_step‚5049 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚73511510 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚298112977640960 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 _step‚413 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 168885026 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚325340098780160 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4231395352576 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚290392056773392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚95393556 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 168905066 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚73511510 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚290391385296276 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚67100400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚42893108 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚298112303275520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚293937278964736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73525524 _runtime‚4031.121437129 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67112694 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4439581667328 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73525524 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4232871156224 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚293937260961792 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚325402797762560 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4217286061812 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24211418 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4216631623848 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42901276 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚320963192155136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚290447105861800 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4232871156224 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚293937260961792 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 168937562 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚437 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 168917496 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚325402612308992 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚293937278964736 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 168917496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 $ _timestamp‚1.788448915649202e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚95412036 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚286230491277312 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 trainer/global_step‚5049 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 168937562 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚435 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42901278 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚95391972 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4439420153856 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4232214780416 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24211853 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889t<[øéåM >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67112694 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4439581667328 P@DeviceStatsMoni-¯þ¥ó$tor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚73525524 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4232871156224 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚293937260961792 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚325402797762560 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4217286061812 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24211418 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4216631623848 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚42901276 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚320963192155136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚290447105861800 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4232871156224 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚293937260961792 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 168937562 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚437 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 168917496 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚325402612308992 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚293937278964736 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 168917496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 $ _timestamp‚1.788448915649202e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚95412036 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚286230491277312 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 trainer/global_step‚5049 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 168937562 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚435 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚42901278 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚95391972 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4439420153856 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4232214780416 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24211853 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚95412036 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚95391972 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚298169475742208 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚73525526 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚298170150120960 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚290447777339124 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚293937278964736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚73525524 _runtime‚4031.121437129ž8!Ó2ê- _wandbruntime‚4031  _runtime‚4031Õî ¹[ëÕ /loss/FBt3/detection_loss‚0.8454633415348722 &loss/total_loss‚47.834366496751386 epoch‚4 trainer/global_step‚5049 /loss/FRt3/detection_loss‚1.6473287984120601 /loss/FBt4/detection_loss‚2.6662344217300413 1loss/FBt4/assignment_loss‚0.22317634876817466 0loss/FRt2/assignment_loss‚3.7986258930630155 2loss/SRqqt1/assignment_loss‚2.3551183844867505 2loss/SRqqt2/assignment_loss‚1.5759648272974625 /loss/FRt4/detection_loss‚2.3874523518390434 2loss/SRqqt4/assignment_loss‚1.6880141885987683 0loss/FBt2/assignment_loss‚0.3899548139402826 -loss/FRt2/detection_loss‚2.39943410621749 1loss/SRqqt2/detection_loss‚1.7910374281583008 1loss/SRqqt3/detection_loss‚1.0556025713449344 /loss/FBt1/detection_loss‚2.8132212396178926 2loss/SRqqt3/assignment_loss‚1.0737781242642086 .loss/FRt1/detection_loss‚3.339773340658708 _step‚415 1loss/SRqqt1/detection_loss‚3.1899965970139754 1loss/FBt3/assignment_loss‚0.23939250472402618 /loss/FRt1/assignment_loss‚4.957764506340027 0loss/FRt3/assignment_loss‚1.4633039017112888 .loss/FBt2/detection_loss‚2.740795992887937 % _timestamp‚1.7884489156510549e+09 _runtime‚4031.122570067 /loss/FRt4/assignment_loss‚2.922088460866795 /loss/FBt1/assignment_loss‚0.530466626182845 1loss/SRqqt4/detection_loss‚1.7403777270924812iåZ8`ìÚ /loss/FBt4/detection_loss‚2.6662344217300413 1loss/FBt4/assignment_loss‚0.22317634876817466 0loss/FRt2/assignment_loss‚3.7986258930630155 2loss/SRqqt1/assignment_loss‚2.3551183844867505 2loss/SRqqt2/assignment_loss‚1.5759648272974625 /loss/FRt4/detection_loss‚2.3874523518390434 2loss/SRqqt4/assignment_loss‚1.6880141885987683 0loss/FBt2/assignment_loss‚0.3899548139402826 -loss/FRt2/detection_loss‚2.39943410621749 1loss/SRqqt2/detection_loss‚1.7910374281583008 1loss/SRqqt3/detection_loss‚1.0556025713449344 /loss/FBt1/detection_loss‚2.8132212396178926 2loss/SRqqt3/assignment_loss‚1.0737781242642086 .loss/FRt1/detection_loss‚3.339773340658708 _step‚415 1loss/SRqqt1/detection_loss‚3.1899965970139754 1loss/FBt3/assignment_loss‚0.23939250472402618 /loss/FRt1/assignment_loss‚4.957764506340027 0loss/FRt3/assignment_loss‚1.4633039017112888 .loss/FBt2/detection_loss‚2.740795992887937 % _timestamp‚1.7884489156510549e+09 _runtime‚4031.122570067 /loss/FRt4/assignment_loss‚2.922088460866795 /loss/FBt1/assignment_loss‚0.530466626182845 1loss/SRqqt4/detection_loss‚1.7403777270924812 /loss/FBt3/detection_loss‚0.8454633415348722 &loss/total_loss‚47.834366496751386 epoch‚4 trainer/global_step‚5049 /loss/FRt3/detection_loss‚1.6473287984120601Ÿ›/@“í‚(:ˆ —¡æÔˈÆþ gpu.0.gpu‚56.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚148.291' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚29.6582 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ù™Sî‚(:È —¡æÔ…Ò®€ disk.rbd0.out‚0) proc.memory.percent‚16.13898277282715% disk.nvme0n1.out‚16396.1220703125 disk.dm-1.in‚0" disk.dm-0.out‚16396.1220703125 disk.sda.out‚ 154.96484375 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda2.in‚0 proc.cpu.threads‚17) disk./.usagePercent‚20.17937685021765 disk.rbd1.out‚ 20.9326171875 disk.sda2.out‚0 disk.sda3.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda1.out‚0# memory_percent‚94.1965103149414 cpu‚25.35242003020668 disk.drbd2265.in‚0 disk.dm-1.out‚ 154.96484375 disk.sda3.out‚ 154.96484375 disk.sda1.in‚0$ disk./.usageGB‚300.6872978210449 disk.nvme0c0n1.in‚0.078125% proc.memory.availableMB‚ 950.84375" proc.memory.rssMB‚ 2644.2109375 disk.sda.in‚0 disk.drbd2265.out‚0 network.sent‚ 1.1611629e+07 network.recv‚ 1.482554e+06 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚16396.1220703125ã,š[“ï‚(:ˆ ¦¡æÔ¹Ëÿü gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.949' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5898 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0nÄøñUð‚(:Ê ¦¡æÔà½Øþ disk.drbd2265.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 cpu‚25.351268284302726 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125% disk.nvme0n1.out‚16452.3056640625" proc.memory.rssMB‚ 2644.2109375 network.recv‚ 1.489875e+06 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚0 disk.dm-1.out‚ 155.6015625$ memory_percent‚94.19803619384766% disk./.usageGB‚300.70348358154297 disk.drbd2266.in‚0" disk.dm-0.out‚16452.3056640625 disk.sda.out‚ 155.59765625 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0% proc.memory.availableMB‚ 950.59375 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚16452.3056640625 disk.sda.in‚0* disk./.usagePercent‚20.180463090185434 disk.sda2.out‚0 disk.rbd1.out‚ 20.9326171875 disk.drbd1716.in‚0) proc.memory.percent‚16.13898277282715 disk.sda3.out‚ 155.6015625 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 network.sent‚ 1.1720902e+07 disk.sda3.in‚0 disk.rbd0.out‚0 disk.sda2.in‚0JDϱžñ‚(:“ µ¡æÔ¤…ý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚136.092' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.218400000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡“ ÀZò‚(:Ï µ¡æÔÛŸóÿ disk.sda.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 20.9951171875 disk.drbd2266.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚16505.52392578125 disk.rbd0.in‚0 proc.cpu.threads‚17* disk./.usagePercent‚20.181330187703907 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 network.recv‚ 1.493293e+06 disk.dm-1.out‚ 156.3046875 disk.drbd1716.out‚0 disk.sda3.out‚ 156.3046875$ memory_percent‚94.19810771942139) proc.memory.percent‚16.13898277282715 cpu‚25.350148959795675 disk.dm-1.in‚0 disk.rbd1.in‚0 disk.sda3.in‚0# disk.dm-0.out‚16505.52392578125" proc.memory.rssMB‚ 2644.2109375% disk./.usageGB‚300.71640396118164 disk.drbd1716.in‚0 disk.drbd2265.out‚0( proc.memory.availableMB‚ 950.58203125 disk.sda2.in‚0& disk.nvme0n1.out‚16505.52392578125 disk.drbd2265.in‚0 disk.sda.out‚ 156.3046875 network.sent‚ 1.1726011e+07 disk.sda1.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125óí5ø2ó- _wandbruntime‚4067  _runtime‚4067ãIóÇôÁ & lr-AdamW/pg1‚9.076152714074621e-05 & lr-AdamW/pg2‚9.076152714074621e-05 trainer/global_step‚5099 % _timestamp‚1.7884489515899174e+09 _runtime‚4067.061530396 _step‚416c 0ÌõÆ & lr-AdamW/pg1‚9.076152714074621e-05 & lr-AdamW/pg2‚9.076152714074621e-05 trainer/global_step‚5099 % _timestamp‚1.7884489515899174e+09 _runtime‚4067.061530396 _step‚416 otž'2ö- _wandbruntime‚4067  _runtime‚4067ŒåZ<Ý'÷×O D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚296698577868288 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚296698577868288 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚96299570 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚5099 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4481716215296 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚300970941023744 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4272381158400 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚323984102240768 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4256650528292 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚296698577868288 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚96299570 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚5099 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4481716215296 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚300970941023744 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4272381158400 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚323984102240768 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4256650528292 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4482422957056 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚324045752978432 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚74226326 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚301028113490432 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 170544348 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24439671 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚296754912904192 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 170564414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚74226324 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚328528151995392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74226324 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4273856962048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚324045729038336 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚5099 $ _timestamp‚1.788448952283227e+09 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67748901 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚293233819206532 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚293233147729208 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 170544348 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚96338088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _runtime‚4067.755355492 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚301028787869184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚67749349 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚96318024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24439225 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚288975697805312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4273856962048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚288975680765952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚96318024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4273200586240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚96338088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43309678 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43309676nøÚë&ûåM J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚96318024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4273200586240 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚96338088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43309678 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43309676 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4482422957056 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚324045752978432 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚74226326 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚301028113490432 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 170544348 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24439671 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚296754912904192 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 170564414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚74226324 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚328528151995392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74226324 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4273856962048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚324045729038336 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚5099 $ _timestamp‚1.788448952283227e+09 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚67748901 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚293233819206532 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚293233147729208 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 170544348 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚96338088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _runtime‚4067.755355492 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚301028787869184 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚67749349 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚96318024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24439225 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚288975697805312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4273856962048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚288975680765952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928¢A¸C2ü- _wandbruntime‚4067  _runtime‚4067ÃV!°]ý× %loss/total_loss‚49.62135423402312 1loss/SRqqt1/assignment_loss‚2.179519763423337 1loss/FBt3/assignment_loss‚0.25144680128589525 _step‚419 /loss/FRt1/assignment_loss‚5.316861007894788 0loss/FRt2/assignment_loss‚3.7128889700199696 0loss/FRt3/assignment_loss‚1.8859477571721346 1loss/SRqqt2/detection_loss‚1.9039935605092482 /loss/FBt3/detection_loss‚0.8253881084200527 _runtime‚4067.756372511 /loss/FRt4/assignment_loss‚3.454458345758154 2loss/SRqqt3/assignment_loss‚1.2814280332052423 .loss/FRt3/detection_loss‚1.663231920029195 1loss/SRqqt1/detection_loss‚2.8311660091082254 trainer/global_step‚5099 2loss/SRqqt2/assignment_loss‚1.7796447198499332 2loss/SRqqt4/assignment_loss‚1.3319366984069347 /loss/FRt1/detection_loss‚3.0232290902308057 /loss/FRt2/detection_loss‚2.1789123403265123 % _timestamp‚1.7884489522848752e+09 0loss/FBt2/assignment_loss‚0.6572321387473494 0loss/FBt4/assignment_loss‚0.2767982315272093 /loss/FRt4/detection_loss‚2.1881840818739953 1loss/SRqqt4/detection_loss‚1.5183102080455193 epoch‚4 1loss/FBt1/assignment_loss‚0.42795307040669284 1loss/SRqqt3/detection_loss‚1.3435482918558752 /loss/FBt1/detection_loss‚2.8907599091529845 /loss/FBt2/detection_loss‚2.4145416915416718 /loss/FBt4/detection_loss‚4.2839734852313995\m™bþÜ epoch‚4 1loss/FBt1/assignment_loss‚0.42795307040669284 1loss/SRqqt3/detection_loss‚1.3435482918558752 /loss/FBt1/detection_loss‚2.8907599091529845 /loss/FBt2/detection_loss‚2.4145416915416718 /loss/FBt4/detection_loss‚4.2839734852313995 %loss/total_loss‚49.62135423402312 1loss/SRqqt1/assignment_loss‚2.179519763423337 1loss/FBt3/assignment_loss‚0.25144680128589525 _step‚419 /loss/FRt1/assignment_loss‚5.316861007894788 0loss/FRt2/assignment_loss‚3.7128889700199696 0loss/FRt3/assignment_loss‚1.8859477571721346 1loss/SRqqt2/detection_loss‚1.9039935605092482 /loss/FBt3/detection_loss‚0.8253881084200527 _runtime‚4067.756372511 /loss/FRt4/assignment_loss‚3.454458345758154 2loss/SRqqt3/assignment_loss‚1.2814280332052423 .loss/FRt3/detection_loss‚1.663231920029195 1loss/SRqqt1/detection_loss‚2.8311660091082254 trainer/global_step‚5099 2loss/SRqqt2/assignment_loss‚1.7796447198499332 2loss/SRqqt4/assignment_loss‚1.3319366984069347 /loss/FRt1/detection_loss‚3.0232290902308057 /loss/FRt2/detection_loss‚2.1789123403265123 % _timestamp‚1.7884489522848752e+09 0loss/FBt2/assignment_loss‚0.6572321387473494 0loss/FBt4/assignment_loss‚0.2767982315272093 /loss/FRt4/detection_loss‚2.1881840818739953 1loss/SRqqt4/detection_loss‚1.5183102080455193£ÉT»“ÿ‚(:ˆ Ä¡æÔÙø©þ gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚148.947' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚29.7894 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0~&O€‚(:Ä Ä¡æԧ܆ÿ disk.nvme0n1.in‚0.078125 disk.sda1.in‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.drbd2080.in‚0# memory_percent‚94.1997766494751( proc.memory.availableMB‚ 950.30859375 disk.sda.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 20.9951171875) proc.memory.percent‚16.13898277282715 disk.sda2.out‚0 disk.dm-1.in‚0 disk.dm-1.out‚ 156.90625 disk.nvme0c0n1.in‚0.078125 cpu‚25.349064168467674 network.recv‚ 1.498417e+06 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚16565.5302734375 disk.drbd2266.out‚0$ disk./.usageGB‚300.7470932006836 disk.drbd2265.out‚0 disk.sda3.out‚ 156.90625 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚16565.5302734375% disk.nvme0n1.out‚16565.5302734375 network.sent‚ 1.1732509e+07 disk.sda.out‚ 156.90625 disk.sda1.out‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0* disk./.usagePercent‚20.183389768316868 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.in‚0" proc.memory.rssMB‚ 2644.2109375 proc.cpu.threads‚17"¹ëQ“‚(:ˆ Ó¡æÔ½äøü gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.721' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.1442 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€RLO‚‚(:Ä Ó¡æÔºÀüþ disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.nvme0n1.in‚0.078125" proc.memory.rssMB‚ 2644.2109375 disk.drbd2265.in‚0 disk.sda3.in‚0 disk.sda1.in‚0$ memory_percent‚94.19543743133545 proc.cpu.threads‚17' disk.nvme0c0n1.out‚16608.5185546875 disk.dm-1.in‚0 disk.drbd1716.out‚0 disk.drbd2266.out‚0" disk.dm-0.out‚16608.5185546875 disk.sda.in‚0 disk.rbd0.in‚0 disk.sda2.in‚0 cpu‚25.347977542564465 network.sent‚ 1.1840246e+07 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 157.234375 disk.rbd1.in‚0) proc.memory.percent‚16.13898277282715$ disk./.usageGB‚300.7678413391113 disk.dm-1.out‚ 157.234375 disk.rbd0.out‚0% disk.nvme0n1.out‚16608.5185546875 disk.sda3.out‚ 157.234375 disk.sda2.out‚0 network.recv‚ 1.504165e+06( disk./.usagePercent‚20.1847821932956 disk.sda1.out‚0 disk.rbd1.out‚ 21.13671875( proc.memory.availableMB‚ 951.01953125 disk.drbd1716.in‚0p¢.ƒ2ƒ- _wandbruntime‚4104  _runtime‚4104&·¨úÇ„Á _step‚420 & lr-AdamW/pg1‚8.993437976831113e-05 & lr-AdamW/pg2‚8.993437976831113e-05 trainer/global_step‚5149 % _timestamp‚1.7884489886629763e+09 _runtime‚4104.134737152ƒ½”˜Ì…Æ trainer/global_step‚5149 % _timestamp‚1.7884489886629763e+09 _runtime‚4104.134737152 _step‚420 & lr-AdamW/pg1‚8.993437976831113e-05 & lr-AdamW/pg2‚8.993437976831113e-05¤ì;…\2†- _wandbruntime‚4104  _runtime‚4104<€ëЇÕO M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚68373156 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚291666000202752 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚299516229810688 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚299516211807744 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚431 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚97245660 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4313366964224 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 172158770 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚331591417800192 D>DeviceStatså¸ÔÆ"Monitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚74913110 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚299516211807744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚74913110 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚421 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚303829578771968 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4524610857984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚5149 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚74913108 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚97225622 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4297485867700 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24663679 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚291665983163392 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4314023326720 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚299516229810688 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚303830253137408 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚74913110 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚299516211807744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚74913110 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚421 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚303829578771968 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4524610857984 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚5149 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚74913108 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚97225622 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4297485867700 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24663679 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚291665983163392 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4314023326720 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚299516229810688 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚303830253137408 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4525635899904 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚291720887293952 _runtime‚4104.826068587 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚303886751238656 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 172191266 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚444 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4298302302664 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74927124 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 172171200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24667771 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4314186392064 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚422 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚331653736113664 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4314186392064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43718078 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚5149 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚68385847 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚97264140 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚331653925761536 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚327128289861632 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚299572582849536 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4314842767872 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚296019189596616 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚296019861073940 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24668213 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43718076 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430ªž)Ë‹æM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4525635899904 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚291720887293952 _runtime‚4104.826068587 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚303886751238656 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 172191266 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚444 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4298302302664 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚74927124 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 172171200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24667771 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4314186392064 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚422 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚331653736113664 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4314186392064 XJDeviceStatsMonitor.o W “În_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚43718078 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚5149 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚68385847 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚97264140 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚331653925761536 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚327128289861632 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚299572582849536 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4314842767872 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚296019189596616 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚296019861073940 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24668213 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚43718076 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚97244076 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4314842767872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚74927126 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4525470192128 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚306688216520192 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚298750182375616 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4567915776000 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚306688890885632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚75613910 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚439 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚302333881753088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚5199 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚98171712 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 173765582 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4355009132544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚302333863750144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44118310 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44118308 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24892651 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24892212 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚334716900024832 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚302333863750144 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚98151674 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4354352770048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚69010961 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚330149152067072 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4355009132544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4338975644864 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚69010520 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4567747957760 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚306688890885632 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚330149176007168 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚294411206730752 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7884490253368745e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚334717091783168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚75613908 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4338321207108 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚4140.808665007¢û‰6 šÜO ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚334716900024832 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚302333863750144 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚98151674 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4354352770048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚69010961 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚330149152067072 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4355009132544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4338975644864 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚69010520 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4567747957760 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚306688890885632 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚330149176007168 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚294411206730752 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7884490253368745e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚334717091783168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚75613908 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4338321207108 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚4140.808665007 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 173785622 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚98171712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚98151674 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚298749510898500 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4354352770048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚294411189691392 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚306688216520192 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚306688216520192 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚298750182375616 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4567915776000 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚306688890885632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚75613910 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚439 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 GADeviceStatsMonFjŽŠ¬itor.on_train_batch_start/active.large_pool.current‚2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚302333881753088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚5199 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚98171712 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 173765582 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4355009132544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚302333863750144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44118310 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44118308 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚24892651 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚24892212©<ö—2›- _wandbruntime‚4141  _runtime‚4141>«uKæ&œàM M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚330210802804736 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚75627924 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚298805231464024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚75627926 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44126478 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24896649 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24897092 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚75627924 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 173798052 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚98190192 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4339137642072 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _step‚426 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69023125 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4355172197888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚98190192 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚294466110861312 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 173818118 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚302390234791936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚330210826744832 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44126476 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚302390216788992 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4355172197888 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚334779598444032 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚298805902941348 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 trainer/global_step‚5199 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 173798052 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 173818118 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚306746063365632 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚306746063365632 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚98170128 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4568771699200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚302390216788992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4355828573696 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 _runtime‚4141.502460363 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚302390234791936®/“ë&åM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚302390234791936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚330210802804736 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚75627924 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚298805231464024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚75627926 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44126478 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚24896649 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚24897092 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚75627924 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 173798052 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚98190192 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4339137642072 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _step‚426 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69023125 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4355172197888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚98190192 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚294466110861312 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 173818118 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚302390234791936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚330210826744832 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44126476 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚302390216788992 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4355172197888 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚334779598444032 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚298805902941348 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 trainer/global_step‚5199 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 173798052 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 173818118 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚306746063365632 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚306746063365632 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚98170128 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4568771699200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚302390216788992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4355828573696 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 _runtime‚4141.502460363ª o¨H2ž- _wandbruntime‚4141  _runtime‚41418‹CwTŸÎ /loss/FRt2/assignment_loss‚3.186879026646517 $ _timestamp‚1.788449026032082e+09 _step‚427 /loss/FRt3/assignment_loss‚1.977709818145503 0loss/FBt4/assignment_loss‚0.4392289072275162 /loss/FRt1/detection_loss‚2.9264660009316037 /loss/FRt3/detection_loss‚1.9785325128747069 /loss/FBt1/detection_loss‚2.8842730578035116 0loss/FBt1/assignment_loss‚0.5294343250698148 0loss/FBt3/assignment_loss‚0.3815091840545839 /loss/FBt2/detection_loss‚1.8941232442855835 .loss/FRt4/assignment_loss‚2.45511556372923 0loss/FBt2/assignment_loss‚0.4101260628505031 .loss/FBt4/detection_loss‚4.650344431400299 1loss/SRqqt1/assignment_loss‚1.912522873053184 1loss/SRqqt3/assignment_loss‚1.108279499976385 /loss/FRt2/detection_loss‚2.2184825941007964 /loss/FBt3/detection_loss‚0.8714219087556209 /loss/FRt4/detection_loss‚1.9895515684403626 _runtime‚4141.503566531 1loss/SRqqt2/assignment_loss‚1.524344821795821 1loss/SRqqt3/detection_loss‚1.2745763293257164 epoch‚4 /loss/FRt1/assignment_loss‚4.380857846566609 2loss/SRqqt4/assignment_loss‚1.4226415465344433 0loss/SRqqt1/detection_loss‚2.698090202533282 1loss/SRqqt2/detection_loss‚1.6485368272662162 1loss/SRqqt4/detection_loss‚1.5506725824541516 %loss/total_loss‚46.31372073582196 trainer/global_step‚5199!žKY Ó /loss/FBt1/detection_loss‚2.8842730578035116 0loss/FBt1/assignment_loss‚0.5294343250698148 0loss/FBt3/assignment_loss‚0.3815091840545839 /loss/FBt2/detection_loss‚1.8941232442855835 .loss/FRt4/assignment_loss‚2.45511556372923 0loss/FBt2/assignment_loss‚0.4101260628505031 .loss/FBt4/detection_loss‚4.650344431400299 1loss/SRqqt1/assignment_loss‚1.912522873053184 1loss/SRqqt3/assignment_loss‚1.108279499976385 /loss/FRt2/detection_loss‚2.2184825941007964 /loss/FBt3/detection_loss‚0.8714219087556209 /loss/FRt4/detection_loss‚1.9895515684403626 _runtime‚4141.503566531 1loss/SRqqt2/assignment_loss‚1.524344821795821 1loss/SRqqt3/detection_loss‚1.2745763293257164 epoch‚4 /loss/FRt1/assignment_loss‚4.380857846566609 2loss/SRqqt4/assignment_loss‚1.4226415465344433 0loss/SRqqt1/detection_loss‚2.698090202533282 1loss/SRqqt2/detection_loss‚1.6485368272662162 1loss/SRqqt4/detection_loss‚1.5506725824541516 %loss/total_loss‚46.31372073582196 trainer/global_step‚5199 /loss/FRt2/assignment_loss‚3.186879026646517 $ _timestamp‚1.788449026032082e+09 _step‚427 /loss/FRt3/assignment_loss‚1.977709818145503 0loss/FBt4/assignment_loss‚0.4392289072275162 /loss/FRt1/detection_loss‚2.9264660009316037 /loss/FRt3/detection_loss‚1.9785325128747069«ãë`3ž¡‚(:“ ¢æÔ…Õöü gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚132.976' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.595200000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÔhÑh¢‚(:Ý ¢æÔ…ôäÿ disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 160.48828125 cpu‚25.343634518340856 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 160.48828125 disk.drbd2080.out‚0% proc.memory.availableMB‚ 951.15625 network.sent‚ 1.1965302e+07 disk.dm-0.out‚ 16824.5078125 disk.sda.out‚ 160.48828125 disk.drbd1716.out‚0 disk.drbd2080.in‚0# memory_percent‚94.1946029663086 network.recv‚ 1.522888e+06* disk./.usagePercent‚20.189015277874265% disk./.usageGB‚300.83091735839844" disk.nvme0n1.out‚ 16824.5078125 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2644.2109375 disk.sda2.in‚0 disk.rbd1.out‚ 21.3408203125 disk.drbd2266.out‚0) proc.memory.percent‚16.13898277282715 proc.cpu.threads‚17 disk.sda2.out‚0$ disk.nvme0c0n1.out‚ 16824.5078125 disk.dm-0.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0Ï‹¯ß“£‚(:ˆ ž¢æÔÂëý gpu.0.gpu‚33.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.406' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.0812 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0CPŠ m¤‚(:â ž¢æÔ´–€ disk.nvme0n1.in‚0.078125 network.sent‚ 1.207469e+07$ disk./.usageGB‚300.8384895324707' disk.nvme0c0n1.out‚16888.4560546875% proc.memory.availableMB‚ 951.15625 cpu‚25.342583104391622 network.recv‚ 1.53043e+06 disk.rbd0.out‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875" disk.dm-0.out‚16888.4560546875 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 21.3408203125 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2266.out‚0% disk.nvme0n1.out‚16888.4560546875 disk.drbd2265.in‚0 disk.sda3.out‚ 161.16015625 disk.rbd0.in‚0" proc.memory.rssMB‚ 2644.2109375 proc.cpu.threads‚17) disk./.usagePercent‚20.18952345282973 disk.sda.in‚ 0.01171875 disk.sda1.in‚0) proc.memory.percent‚16.13898277282715 disk.dm-0.in‚0.078125 disk.sda.out‚ 161.16015625# memory_percent‚94.1946029663086 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 161.16015625 disk.drbd1716.out‚0 disk.sda1.out‚0Ìþì2¥- _wandbruntime‚4177  _runtime‚4177HVjǦÁ _step‚428 & lr-AdamW/pg1‚8.826363364747533e-05 & lr-AdamW/pg2‚8.826363364747533e-05 trainer/global_step‚5249 % _timestamp‚1.7884490624118574e+09 _runtime‚4177.883463545¿[~¥Ì§Æ & lr-AdamW/pg1‚8.826363364747533e-05 & lr-AdamW/pg2‚8.826363364747533e-05 trainer/global_step‚5249 % _timestamp‚1.7884490624118574e+09 _runtime‚4177.883463545 _step‚428¬N X’2¨- _wandbruntime‚4177  _runtime‚4177½Ù|À©×O D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚309547528633856 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚337842763274752 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚333231688950272 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚76314708 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.largÏ^ãKØe_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4379156546516 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚76314710 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚305151533695488 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4395338575872 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚337842579905024 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚309546854268416 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44526708 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4395994938368 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚305151533695488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4395338575872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25120920 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4611050384384 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚305151515692544 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚99077726 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7884490624125822e+09@R×â'ªÜO LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4395338575872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25120920 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4611050384384 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚305151515692544 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚99077726 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 % _timestamp‚1.7884490624125822e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚309547528633856 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚337842763274752 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚333231688950272 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚76314708 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4379156546516 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚76314710 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚305151533695488 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4395338575872 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚337842579905024 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚309546854268416 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44526708 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4395994938368 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚305151533695488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038­Šá42«- _wandbruntime‚4178  _runtime‚4178EAÜæ&¬àM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4396814379520 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚297211300349952 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚305207886734336 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚76328726 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 175444970 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚99096180 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚305207886734336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4396814379520 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 175424904 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25125873 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚99096180 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚305207868731392 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚76328726 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚309604026735104 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 175444970 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44534878 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25125420 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69660296 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4396158003712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4611751403008 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚76328724 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4396158003712 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚305207868731392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 175424904 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚4178.572393889 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚76328724 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚453 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚337905091090944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4611915013632 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _step‚430 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚99116244 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚297211317389312 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚309604701113856 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚333293339687936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44534876 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚99116244 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 175444970 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚99096180 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚305207886734336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4396814379520 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 175424904 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25125873 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚99096180 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚305207868731392 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚76328726 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚309604026735104 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 175444970 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44534878 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25125420 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚69660296 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4396158003712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4611751403008 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚76328724 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4396158003712 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚305207868731392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 175424904 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚4178.572393889 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚76328724 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚453 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚337905091090944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4611915013632 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _step‚430 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚99116244 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚297211317389312 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚309604701113856 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚333293339687936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44534876 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928®‰Jë2®- _wandbruntime‚4178  _runtime‚4178SµcW¯Ñ 2loss/SRqqt4/assignment_loss‚1.1843418380511659 % _timestamp‚1.7884490631019127e+09 _step‚431 .loss/FRt1/detection_loss‚2.797599175037482 .loss/FBt2/detection_loss‚2.034790487516494 .loss/FBt4/detection_loss‚3.118933563431104 /loss/FRt1/assignment_loss‚4.317879511759831 0loss/SRqqt4/detection_loss‚1.906160797391619 trainer/global_step‚5249 2loss/SRqqt1/assignment_loss‚2.3203091657161714 1loss/SRqqt3/detection_loss‚1.5500849894827828 /loss/FRt4/detection_loss‚2.3365987747214563 .loss/SRqqt2/detection_loss‚2.0659244344348 /loss/FBt3/detection_loss‚0.7870773856823401 0loss/FRt2/assignment_loss‚3.6876782515110116 0loss/FRt4/assignment_loss‚3.2114219249680986 2loss/SRqqt2/assignment_loss‚1.6683144478925638 /loss/FBt1/detection_loss‚2.5923921328324537 0loss/FRt3/assignment_loss‚2.1768960042433307 2loss/SRqqt3/assignment_loss‚1.3564429023074653 0loss/FBt1/assignment_loss‚0.7138848774708234 /loss/FRt2/detection_loss‚2.0208987654783788 1loss/SRqqt1/detection_loss‚2.7654009866714477 %loss/total_loss‚47.57927612033674 _runtime‚4178.573407692 0loss/FBt2/assignment_loss‚0.4900856101794705 1loss/FBt3/assignment_loss‚0.29463130085729516 0loss/FBt4/assignment_loss‚0.3238185859873208 .loss/FRt3/detection_loss‚1.857710206711834 epoch‚47­wô\°Ö 2loss/SRqqt1/assignment_loss‚2.3203091657161714 1loss/SRqqt3/detection_loss‚1.5500849894827828 /loss/FRt4/detection_loss‚2.3365987747214563 .loss/SRqqt2/detection_loss‚2.0659244344348 /loss/FBt3/detection_loss‚0.7870773856823401 0loss/FRt2/assignment_loss‚3.6876782515110116 0loss/FRt4/assignment_loss‚3.2114219249680986 2loss/SRqqt2/assignment_loss‚1.6683144478925638 /loss/FBt1/detection_loss‚2.5923921328324537 0loss/FRt3/assignment_loss‚2.1768960042433307 2loss/SRqqt3/assignment_loss‚1.3564429023074653 0loss/FBt1/assignment_loss‚0.7138848774708234 /loss/FRt2/detection_loss‚2.0208987654783788 1loss/SRqqt1/detection_loss‚2.7654009866714477 %loss/total_loss‚47.57927612033674 _runtime‚4178.573407692 0loss/FBt2/assignment_loss‚0.4900856101794705 1loss/FBt3/assignment_loss‚0.29463130085729516 0loss/FBt4/assignment_loss‚0.3238185859873208 .loss/FRt3/detection_loss‚1.857710206711834 epoch‚4 2loss/SRqqt4/assignment_loss‚1.1843418380511659 % _timestamp‚1.7884490631019127e+09 _step‚431 .loss/FRt1/detection_loss‚2.797599175037482 .loss/FBt2/detection_loss‚2.034790487516494 .loss/FBt4/detection_loss‚3.118933563431104 /loss/FRt1/assignment_loss‚4.317879511759831 0loss/SRqqt4/detection_loss‚1.906160797391619 trainer/global_step‚5249¯õùpf“±‚(:ˆ ­¢æÔ†þü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.413' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6826 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚01ÿëp²‚(:å ­¢æÔ‘ņÿ network.recv‚ 1.533685e+06 disk.sda.in‚ 0.01171875 disk.sda2.in‚0% disk.nvme0n1.out‚16934.1357421875 disk.dm-1.out‚ 162.0078125 disk.drbd2080.out‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 cpu‚25.341612849409206 disk.rbd0.out‚0 disk.drbd2080.in‚0 disk.rbd1.in‚0 network.sent‚ 1.2079835e+07 disk.sda2.out‚0$ memory_percent‚94.19722557067871" proc.memory.rssMB‚ 2644.2109375$ disk./.usageGB‚300.8508605957031' disk.nvme0c0n1.out‚16934.1357421875 disk.rbd0.in‚0" disk.dm-0.out‚16934.1357421875 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda.out‚ 162.0078125' proc.memory.availableMB‚ 950.7265625) proc.memory.percent‚16.13898277282715 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 162.0078125 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.rbd1.out‚ 21.4033203125 disk.drbd1716.in‚0* disk./.usagePercent‚20.190353685263275 disk.dm-1.in‚ 0.01171875q,+b“³‚(:ˆ ¼¢æÔÌÍ™ý gpu.0.gpu‚43.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚169.724' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.9448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0gfol´‚(:á ¼¢æÔ»Ô—ÿ disk.dm-1.out‚ 162.6171875 proc.cpu.threads‚17 disk.sda3.out‚ 162.6171875 disk.rbd0.in‚0 disk.sda.out‚ 162.6171875 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0) proc.memory.percent‚16.13898277282715 network.sent‚ 1.218881e+07$ disk./.usageGB‚300.8458824157715 disk.rbd1.out‚ 21.4892578125 cpu‚25.340577002603755' disk.nvme0c0n1.out‚16990.5048828125 disk.rbd0.out‚0 disk.sda1.in‚0 disk.drbd2265.in‚0' proc.memory.availableMB‚ 950.7265625" proc.memory.rssMB‚ 2644.2109375 disk.dm-0.in‚0.078125 disk.sda2.out‚0 network.recv‚ 1.5411e+06 disk.drbd2080.in‚0 disk.sda2.in‚0$ memory_percent‚94.19722557067871 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0) disk./.usagePercent‚20.19001959543109 disk.drbd2266.in‚0" disk.dm-0.out‚16990.5048828125 disk.sda1.out‚0% disk.nvme0n1.out‚16990.5048828125˜RÙ2µ- _wandbruntime‚4214  _runtime‚42147Ownö½ $ lr-AdamW/pg2‚8.7420383080062e-05 trainer/global_step‚5299 % _timestamp‚1.7884490992932143e+09 _runtime‚4214.764854074 _step‚432 $ lr-AdamW/pg1‚8.7420383080062e-05^ÈÂûÈ·Â $ lr-AdamW/pg1‚8.7420383080062e-05 $ lr-AdamW/pg2‚8.7420383080062e-05 trainer/global_step‚5299 % _timestamp‚1.7884490992932143e+09 _runtime‚4214.764854074 _step‚432°{§2¸- _wandbruntime‚4214  _runtime‚4214àå\á'¹ÛO UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 177039326 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚299901602747392 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚312405492016640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚307969185637888 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44935108 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚299901619786752 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4654066072576 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚304321594633316 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4419991885924 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 177019286 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚307969185637888 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚312406166382080 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 100003778 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25349117 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4436980744192 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚336314249773568 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚77015508 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 100023816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚77015510 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44935110 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚307969167634944 _runtime‚4214.765783839 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 100003778 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4436324381696 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25349551 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚70284225 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4653906642944 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚77015508 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚436 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _step‚433 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 100023816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚340968132476416 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚336314225833472 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚304322266110432 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚312405492016640 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚307969167634944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚70284661 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4436324381696 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚312406166382080 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414ïÂXSæ'ºàO eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚336314249773568 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚77015508 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 100023816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚77015510 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚44935110 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚307969167634944 _runtime‚4214.765783839 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 100003778 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4436324381696 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25349551 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚70284225 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4653906642944 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚77015508 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚436 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _step‚433 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 100023816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚340968132476416 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚336314225833472 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚304322266110432 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚312405492016640 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚307969167634944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚70284661 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4436324381696 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚312406166382080 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 177039326 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚299901602747392 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚312405492016640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚307969185637888 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚44935108 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚299901619786752 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4654066072576 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚304321594633316 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4419991885924 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 177019286 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚307969185637888 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚312406166382080 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 100003778 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25349117 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4436980744192 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384±1Z _“»‚(:ˆ Ë¢æÔ鼊þ gpu.0.gpu‚36.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.013' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0026 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ðÔip¼‚(:å Ë¢æÔÇÏ«ÿ disk.dm-1.in‚ 0.01171875 cpu‚25.339489026464836 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚17035.93798828125 disk.sda.out‚ 163.171875 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0) proc.memory.percent‚16.13898277282715 proc.cpu.threads‚17 disk.sda1.in‚0$ memory_percent‚94.19715404510498 disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚17035.93798828125" proc.memory.rssMB‚ 2644.2109375% disk./.usageGB‚300.85374450683594 disk.drbd1716.in‚0 disk.sda3.out‚ 163.171875 disk.rbd1.out‚ 21.7470703125( proc.memory.availableMB‚ 950.73828125* disk./.usagePercent‚20.190547226959158 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 network.sent‚ 1.2193995e+07 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.dm-1.out‚ 163.171875 disk.dm-0.in‚0.078125 network.recv‚ 1.5445e+06 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.rbd0.in‚0# disk.dm-0.out‚17035.93798828125 disk.sda.in‚ 0.01171875VéÕj2½- _wandbruntime‚4215  _runtime‚4215Xî!оäM EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚5299 $ _timestamp‚1.788449099977839e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 100022232 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25353656 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _step‚434 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4437800185344 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 177071822 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 177071822 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚450 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4421462758852 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚77029526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 100042296 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚336375900511232 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44943276 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4420808320888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚70297384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚341030822248960 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚308025520673792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚312462664483328 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚308025538676736 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚77029526 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4654921737728 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚77029524 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚308025520673792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚341030628406784 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚336375876571136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 177051756 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚304377986676164 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44943278 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25354106 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚77029524 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70296932 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚304377315198840 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4437143809536 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 100022232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360}’Xï&¿éM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚44943276 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4420808320888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚70297384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚341030822248960 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚308025520673792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚312462664483328 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚308025538676736 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚77029526 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4654921737728 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚77029524 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚308025520673792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚341030628406784 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚336375876571136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 177051756 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚304377986676164 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚44943278 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25354106 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚77029524 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70296932 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚304377315198840 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4437143809536 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 100022232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚5299 $ _timestamp‚1.788449099977839e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 100022232 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25353656 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _step‚434 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4437800185344 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 177071822 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 177071822 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚450 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4421462758852 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚77029526 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 100042296 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚336375900511232 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872²%Di2À- _wandbruntime‚4215  _runtime‚4215Õ|„TÁÎ %loss/total_loss‚46.90358710796673 1loss/SRqqt1/assignment_loss‚2.223878866652163 1loss/SRqqt2/assignment_loss‚1.638929426178055 /loss/SRqqt4/detection_loss‚1.81868909007194 .loss/FBt1/detection_loss‚2.566796524184091 /loss/FRt4/assignment_loss‚3.611446186900139 2loss/SRqqt3/assignment_loss‚1.2472928242614636 0loss/FBt2/assignment_loss‚0.6120393412808577 1loss/FBt4/assignment_loss‚0.40158424992114305 /loss/FRt3/detection_loss‚1.7133586592340873 0loss/SRqqt1/detection_loss‚2.144505705775284 1loss/SRqqt2/detection_loss‚1.6598513047650176 .loss/FRt2/detection_loss‚2.076550933329955 .loss/FRt1/detection_loss‚3.110760224717004 % _timestamp‚1.7884490999804842e+09 0loss/FRt1/assignment_loss‚4.8182832683835715 .loss/FBt2/detection_loss‚2.253429189324379 trainer/global_step‚5299 _runtime‚4215.452014214 1loss/SRqqt3/detection_loss‚1.1476230767483895 /loss/FBt3/detection_loss‚0.8511415470825279 /loss/FRt2/assignment_loss‚3.655558036721271 _step‚435 0loss/FRt3/assignment_loss‚1.6527109338065324 1loss/SRqqt4/assignment_loss‚1.594471172813107 .loss/FRt4/detection_loss‚1.980734646320343 epoch‚4 0loss/FBt1/assignment_loss‚0.5978188988016038 1loss/FBt3/assignment_loss‚0.23887437566328298 /loss/FBt4/detection_loss‚3.2872586250305176«×Þ%YÂÓ 0loss/FRt3/assignment_loss‚1.6527109338065324 1loss/SRqqt4/assignment_loss‚1.594471172813107 .loss/FRt4/detection_loss‚1.980734646320343 epoch‚4 0loss/FBt1/assignment_loss‚0.5978188988016038 1loss/FBt3/assignment_loss‚0.23887437566328298 /loss/FBt4/detection_loss‚3.2872586250305176 %loss/total_loss‚46.90358710796673 1loss/SRqqt1/assignment_loss‚2.223878866652163 1loss/SRqqt2/assignment_loss‚1.638929426178055 /loss/SRqqt4/detection_loss‚1.81868909007194 .loss/FBt1/detection_loss‚2.566796524184091 /loss/FRt4/assignment_loss‚3.611446186900139 2loss/SRqqt3/assignment_loss‚1.2472928242614636 0loss/FBt2/assignment_loss‚0.6120393412808577 1loss/FBt4/assignment_loss‚0.40158424992114305 /loss/FRt3/detection_loss‚1.7133586592340873 0loss/SRqqt1/detection_loss‚2.144505705775284 1loss/SRqqt2/detection_loss‚1.6598513047650176 .loss/FRt2/detection_loss‚2.076550933329955 .loss/FRt1/detection_loss‚3.110760224717004 % _timestamp‚1.7884490999804842e+09 0loss/FRt1/assignment_loss‚4.8182832683835715 .loss/FBt2/detection_loss‚2.253429189324379 trainer/global_step‚5299 _runtime‚4215.452014214 1loss/SRqqt3/detection_loss‚1.1476230767483895 /loss/FBt3/detection_loss‚0.8511415470825279 /loss/FRt2/assignment_loss‚3.655558036721271 _step‚435³ô3+“Â(:ˆ Ú¢æÔ®Øýü gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.332' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4664 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0’™vÄ‚(:ë Ú¢æÔ¹”ÿ disk.drbd2080.out‚0# disk.dm-0.out‚17091.36279296875 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 network.recv‚ 1.55147e+06$ disk./.usageGB‚300.8633689880371& disk.nvme0n1.out‚17091.36279296875 disk.dm-1.out‚ 164.17578125 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0) proc.memory.percent‚16.13900661468506 cpu‚25.338471794525468 network.sent‚ 1.2292088e+07 disk.drbd1716.in‚0 disk.drbd2266.in‚0# proc.memory.rssMB‚ 2644.21484375* disk./.usagePercent‚20.191193133968046 disk.sda.out‚ 164.17578125 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd2265.in‚0 proc.cpu.threads‚17( disk.nvme0c0n1.out‚17091.36279296875 disk.rbd1.in‚0 disk.sda3.out‚ 164.17578125' proc.memory.availableMB‚ 950.6796875 disk.rbd0.out‚0 disk.drbd2265.out‚0$ memory_percent‚94.19751167297363 disk.rbd1.out‚ 21.7470703125¥| “Å‚(:ˆ é¢æÔÍ¥Åý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.654' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9308 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ý aÉtÆ‚(:é é¢æÔŸ¦’€ disk.rbd1.out‚ 21.8955078125 proc.cpu.threads‚17( disk.nvme0c0n1.out‚17138.39794921875 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0$ memory_percent‚94.19753551483154# proc.memory.rssMB‚ 2644.21484375) proc.memory.percent‚16.13900661468506 disk.sda.out‚ 164.69140625 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.sda2.in‚0( proc.memory.availableMB‚ 950.67578125( disk./.usagePercent‚20.1915374641016 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 cpu‚25.33750572562107 disk.sda3.out‚ 164.69140625 disk.drbd2266.out‚0# disk.dm-0.out‚17138.39794921875 disk.dm-1.out‚ 164.69140625 network.sent‚ 1.2389067e+07 network.recv‚ 1.55679e+06$ disk./.usageGB‚300.8684997558594 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.drbd1716.out‚0& disk.nvme0n1.out‚17138.39794921875 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875!+;L2Ç- _wandbruntime‚4251  _runtime‚4251àИ¥ÇÈÁ % _timestamp‚1.7884491362195287e+09 _runtime‚4251.691139716 _step‚436 & lr-AdamW/pg1‚8.657211479008814e-05 & lr-AdamW/pg2‚8.657211479008814e-05 trainer/global_step‚5349þ.ÁÈÌÉÆ _runtime‚4251.691139716 _step‚436 & lr-AdamW/pg1‚8.657211479008814e-05 & lr-AdamW/pg2‚8.657211479008814e-05 trainer/global_step‚5349 % _timestamp‚1.7884491362195287e+09´£ßw22Ê- _wandbruntime‚4251  _runtime‚42515š÷¯ËÚO E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4477966550016 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 100929830 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25578105 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4697223847424 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚315264129764864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚425 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4697068612096 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚307108307977840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4477310187520 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179175424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚310786819577344 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235328 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚70921615 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 178646138 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4461481663088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 178666178 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45343510 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚302646826314752 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚5349 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4477310187520 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚310786837580288 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚77716310 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚344094010504192 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚339396786656768 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4460827225332 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4477966550016 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonito ­ùD1 r.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _runtime‚ 4251.69221785 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚70921188 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚77716310 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 100949868 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚344093831328768 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚339396762716672 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚302646809275392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚427 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚310786837580288 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 100949868 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430µ‹Žå'ÌßO E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4477966550016 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _runtime‚ 4251.69221785 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚70921188 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚77716310 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 100949868 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚344093831328768 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚339396762716672 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚302646809275392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚427 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚310786837580288 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 100949868 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4477966550016 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 100929830 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25578105 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4697223847424 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚315264129764864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚425 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4697068612096 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚307108307977840 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4477310187520 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179175424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚310786819577344 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235328 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚70921615 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 178646138 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4461481663088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 178666178 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45343510 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚302646826314752 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚5349 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4477310187520 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚310786837580288 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚77716310 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚344094010504192 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚339396786656768 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4460827225332µ(½2Í- _wandbruntime‚4252  _runtime‚4252¨ý–hë&ÎåM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚315321302231552 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70933846 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25582597 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 178698674 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚77730326 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4462298098260 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45351678 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚339458437394432 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 178678608 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4478785991168 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4698086560768 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4461643660296 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 100948284 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚4252.388865908 _step‚438 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚302701713405952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4478129615360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4478129615360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4478785991168 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚310843172616192 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 178678608 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚344156332210176 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚344156523955200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 100968348 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚427 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚307164028543572 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚315321976610304 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 100948284 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45351676 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4697918755840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884491369168882e+09 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4478785991168 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4698086560768 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4461643660296 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 100948284 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚4252.388865908 _step‚438 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚302701713405952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4478129615360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4478129615360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4478785991168 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚310843172616192 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 178678608 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚344156332210176 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚344156523955200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 100968348 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚427 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚307164028543572 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚315321976610304 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 100948284 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚45351676 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4697918755840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884491369168882e+09 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚70933846 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚25582597 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 178698674 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚77730326 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4462298098260 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45351678 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚339458437394432 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.sma¥ÈJFll_pool.peak‚ 268325376 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 178678608 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 180293030 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚78417110 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚313604489522688 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚78417108 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 180272990 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚318123441878528 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚342479299599872 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚318122767513088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4740105114624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚71558870 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚309893678368132 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _step‚441 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4518295993344 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚425 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚347219585987072 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4740262447104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 101875920 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25806537 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚347219404714496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚313604471519744 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4518952355840 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚78417110 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45751908 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚318122767513088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚318123441878528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚313604489522688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45751910 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚309894349845248 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4502317002496#­&æ'ÜàO O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚305392015803392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚313604471519744 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚71558445 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚5399 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4518295993344 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚25806960 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚342479323539968 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 101855882 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚423 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚71558870 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚309893678368132 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _step‚441 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4518295993344 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚425 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚347219585987072 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4740262447104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 101875920 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚25806537 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚347219404714496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚313604471519744 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4518952355840 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚78417110 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚45751908 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚318122767513088 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚318123441878528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚313604489522688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚45751910 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚309894349845248 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4502317002496 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 _runtime‚4288.671428811 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4518952355840 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 101875920 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 101855882 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚78417108 % _timestamp‚1.7884491731996298e+09 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 180293030 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚78417110 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚313604489522688 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚78417108 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 180272990 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚318123441878528 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚342479299599872 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚318122767513088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4740105114624 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 J2Ý- _wandbruntime‚4289  _runtime‚4289+>§ÓwÞåM FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 180325526 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚347282102973952 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4503133437668 trainer/global_step‚5399 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚71571192 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25811116 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚342540974277632 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 _runtime‚4289.368387946 S?DeviceStatsMonitor.on_train_batch_end/allocated_b 8¥htytes.all.freed‚318179939979776 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45760078 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚342540950337536 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4741128696320 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4519771796992 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚309950070410980 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚313660824558592 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 180305460 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚78431124 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4519115421184 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚313660842561536 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚309949398933656 % _timestamp‚1.7884491738963654e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚318180614358528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚427 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚78431124 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 101894400 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚425 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚78431126 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 101874336 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚78431126 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 àýdð&ßêM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚313660842561536 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚309949398933656 % _timestamp‚1.7884491738963654e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚318180614358528 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚427 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚78431124 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 101894400 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚425 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚78431126 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 101874336 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚78431126 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 180325526 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚347282102973952 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4503133437668 trainer/global_step‚5399 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚71571192 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚25811116 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚342540974277632 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 _runtime‚4289.368387946 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚318179939979776 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚45760078 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚342540950337536 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4741128696320 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4519771796992 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚309950070410980 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚313660824558592 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 180305460 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚78431124 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4519115421184 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0ºÿú 22à- _wandbruntime‚4289  _runtime‚4289¶Í™áVáÐ /loss/FRt2/detection_loss‚2.1821605137416293 epoch‚4 -loss/FRt1/assignment_loss‚5.4588316822052 1loss/FBt3/assignment_loss‚0.20153905552317847 1loss/SRqqt1/detection_loss‚2.9276430583000184 0loss/FBt1/assignment_loss‚0.5570879429530383 /loss/FRt1/detection_loss‚3.0730330777168273 2loss/SRqqt4/assignment_loss‚1.5646227955479513 0loss/FBt4/assignment_loss‚0.2299372897425201 .loss/FRt4/detection_loss‚2.237511009640164 /loss/FBt1/detection_loss‚2.4736870654991696 /loss/FBt3/detection_loss‚0.9335937038627699 %loss/total_loss‚47.19348622111906 /loss/FRt3/assignment_loss‚1.711000261398462 /loss/FRt4/assignment_loss‚3.064547492398156 1loss/SRqqt2/assignment_loss‚1.702608455832188 .loss/FBt2/detection_loss‚2.651490173556588 trainer/global_step‚5399 _runtime‚4289.369422108 2loss/SRqqt1/assignment_loss‚2.4114543628692626 /loss/FRt3/detection_loss‚1.8538652048088038 /loss/FBt4/detection_loss‚2.7184335231781005 _step‚443 2loss/SRqqt3/assignment_loss‚1.0272183168522069 1loss/SRqqt3/detection_loss‚0.8893277546469595 /loss/FBt2/assignment_loss‚0.557488809931623 0loss/SRqqt2/detection_loss‚1.740593403004683 1loss/SRqqt4/detection_loss‚1.7749721512875773 % _timestamp‚1.7884491738979094e+09 0loss/FRt2/assignment_loss‚3.2508391166219908B×Ô[âÕ 2loss/SRqqt3/assignment_loss‚1.0272183168522069 1loss/SRqqt3/detection_loss‚0.8893277546469595 /loss/FBt2/assignment_loss‚0.557488809931623 0loss/SRqqt2/detection_loss‚1.740593403004683 1loss/SRqqt4/detection_loss‚1.7749721512875773 % _timestamp‚1.7884491738979094e+09 0loss/FRt2/assignment_loss‚3.2508391166219908 /loss/FRt2/detection_loss‚2.1821605137416293 epoch‚4 -loss/FRt1/assignment_loss‚5.4588316822052 1loss/FBt3/assignment_loss‚0.20153905552317847 1loss/SRqqt1/detection_loss‚2.9276430583000184 0loss/FBt1/assignment_loss‚0.5570879429530383 /loss/FRt1/detection_loss‚3.0730330777168273 2loss/SRqqt4/assignment_loss‚1.5646227955479513 0loss/FBt4/assignment_loss‚0.2299372897425201 .loss/FRt4/detection_loss‚2.237511009640164 /loss/FBt1/detection_loss‚2.4736870654991696 /loss/FBt3/detection_loss‚0.9335937038627699 %loss/total_loss‚47.19348622111906 /loss/FRt3/assignment_loss‚1.711000261398462 /loss/FRt4/assignment_loss‚3.064547492398156 1loss/SRqqt2/assignment_loss‚1.702608455832188 .loss/FBt2/detection_loss‚2.651490173556588 trainer/global_step‚5399 _runtime‚4289.369422108 2loss/SRqqt1/assignment_loss‚2.4114543628692626 /loss/FRt3/detection_loss‚1.8538652048088038 /loss/FBt4/detection_loss‚2.7184335231781005 _step‚443»W$I"“ã‚(:ˆ –£æÔ¹ë„ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.297' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.6594 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0°Dôárä‚(:ç –£æÔ§¶ÿ disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0% disk./.usageGB‚300.89234161376953 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 166.7578125 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 network.sent‚ 1.2510146e+07( disk./.usagePercent‚20.1931375111906 disk.sda1.in‚0( proc.memory.availableMB‚ 950.60546875) proc.memory.percent‚16.13900661468506 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0# proc.memory.rssMB‚ 2644.21484375 disk.rbd0.out‚0# disk.dm-0.out‚17292.62744140625 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 22.123046875 disk.sda3.out‚ 166.7578125$ memory_percent‚94.19796466827393 cpu‚25.33446628787849 disk.sda.out‚ 166.7578125 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 proc.cpu.threads‚17 network.recv‚ 1.572765e+06& disk.nvme0n1.out‚17292.62744140625 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚17292.62744140625 disk.sda2.out‚0 disk.rbd1.in‚0²|å‚(:’ ¥£æÔýÏý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚168.414' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚33.68279999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Òótqæ‚(:æ ¥£æÔ§Ÿ†ÿ( disk.nvme0c0n1.out‚17337.60791015625 disk.sda3.in‚ 0.01171875 disk.rbd1.out‚ 22.185546875& disk.nvme0n1.out‚17337.60791015625) proc.memory.percent‚16.13900661468506 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 167.234375# disk.dm-0.out‚17337.60791015625 disk.sda.out‚ 167.234375 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 167.234375 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.sda2.out‚0( proc.memory.availableMB‚ 950.55859375* disk./.usagePercent‚20.193889149311133 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0# proc.memory.rssMB‚ 2644.21484375 network.recv‚ 1.578451e+06 disk.drbd1716.out‚0 network.sent‚ 1.2617856e+07 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875$ memory_percent‚94.19825077056885$ disk./.usageGB‚300.9035415649414 disk.sda1.in‚0 cpu‚25.333423661203682ìFûþ“ç‚(:ˆ ´£æÔï×–ý gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.509' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.7018 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚08W~\è‚(:Ñ ´£æÔÛþ‚ÿ disk.rbd0.in‚0 disk.sda2.out‚0' disk.nvme0c0n1.out‚17399.8701171875 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.dm-0.in‚0.078125$ proc.memory.availableMB‚950.3125# proc.memory.rssMB‚ 2644.21484375% disk./.usageGB‚300.90958404541016 disk.sda2.in‚0 disk.drbd2265.out‚0 network.sent‚ 1.2624324e+07 disk.rbd1.in‚0 disk.rbd1.out‚ 22.185546875 disk.drbd1716.out‚0 disk.drbd2080.in‚0" disk.dm-0.out‚17399.8701171875 cpu‚25.33244649514545 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0) disk./.usagePercent‚20.19429466524537 disk.sda1.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚167.75 disk.sda.in‚ 0.01171875 disk.sda.out‚167.75$ memory_percent‚94.19975280761719% disk.nvme0n1.out‚17399.8701171875 disk.dm-1.out‚167.75 disk.drbd2266.in‚0) proc.memory.percent‚16.13900661468506 proc.cpu.threads‚17 network.recv‚ 1.58374e+06 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875D',ë2é- _wandbruntime‚4325  _runtime‚4325’·»ØÇêÁ & lr-AdamW/pg1‚8.486123316266528e-05 & lr-AdamW/pg2‚8.486123316266528e-05 trainer/global_step‚5449 % _timestamp‚1.7884492102983496e+09 _runtime‚4325.770043141 _step‚444ÑE€PÌëÆ _step‚444 & lr-AdamW/pg1‚8.486123316266528e-05 & lr-AdamW/pg2‚8.486123316266528e-05 trainer/global_step‚5449 % _timestamp‚1.7884492102983496e+09 _runtime‚4325.770043141¼ؾ‡42ì- _wandbruntime‚4325  _runtime‚43258ß)ÿá'íÛO XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4783533419520 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26035506 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚308137239370752 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚316422123462144 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _step‚445 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 102781934 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚320981405261312 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚46160308 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚312679720235540 trainer/global_step‚5449 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚345561836483072 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚320982079626752 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚79117908 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4559281799168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4559281799168 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚72196268 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚320981405261312 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚46160308 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚312679720235540 trainer/global_step‚5449 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚345561836483072 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚320982079626752 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚79117908 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4559281799168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4559281799168 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚72196268 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4783533419520 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26035506 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚308137239370752 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚316422123462144 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _step‚445 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 102781934 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚316478494503936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26040133 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚464 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 102800388 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚350407892956672 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 $ _timestamp‚1.788449210988317e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚446 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _runtime‚4326.460520905 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚321038577728000 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚316478494503936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 181952378 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72209073 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚316478476500992 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4560757602816 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚312735440801064 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72208609 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚308192126461952 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 181932312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚46168478 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 181952378 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚5449 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚316478476500992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4784381795840 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚79131926 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4543968777076 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4784211893760 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4560757602816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚321039252106752 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 102820452 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 181932312 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 102820452 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0n,áðï&ñéM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚316478476500992 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4560757602816 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚312735440801064 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72208609 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚308192126461952 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 181932312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚46168478 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 181952378 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚5449 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚316478476500992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4784381795840 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚79131926 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4543968777076 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4784211893760 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4560757602816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚321039252106752 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 102820452 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 181932312 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 102820452 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚316478494503936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26040133 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚464 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 102800388 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚350407892956672 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 $ _timestamp‚1.788449210988317e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚446 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _runtime‚4326.460520905 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚321038577728000 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚316478494503936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 181952378 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72209073¾#ýM®2ò- _wandbruntime‚4326  _runtime‚4326!‘‘\WóÑ % _timestamp‚1.7884492109899628e+09 _runtime‚4326.461484534 0loss/FRt4/assignment_loss‚2.9342671940724054 1loss/FBt3/assignment_loss‚0.21402082893267116 .loss/FRt3/detection_loss‚1.648828273303196 2loss/SRqqt2/assignment_loss‚1.4075100870560044 0loss/FBt2/assignment_loss‚0.3057563235228632 1loss/SRqqt3/detection_loss‚1.0896484214191635 1loss/SRqqt3/assignment_loss‚1.000711322024775 /loss/FBt3/detection_loss‚0.6778740182791542 _step‚447 1loss/SRqqt1/assignment_loss‚2.306918515608861 0loss/SRqqt2/detection_loss‚1.815163232062174 epoch‚4 /loss/FRt2/detection_loss‚1.9706316091293512 1loss/SRqqt4/detection_loss‚1.6276666702968734 .loss/FBt2/detection_loss‚4.264806091785431 .loss/FBt4/detection_loss‚3.840959846973419 %loss/total_loss‚48.64891007459618 /loss/FRt1/assignment_loss‚4.369876098632813 0loss/FRt3/assignment_loss‚1.7686523570389043 1loss/SRqqt4/assignment_loss‚1.509626682382077 /loss/FBt1/assignment_loss‚0.455564135064166 /loss/FRt4/detection_loss‚2.2928861488898593 /loss/FRt1/detection_loss‚3.4673613011837006 /loss/FBt1/detection_loss‚2.4742157141367596 trainer/global_step‚5449 0loss/FBt4/assignment_loss‚0.7334896326065063 0loss/FRt2/assignment_loss‚3.7826568165490793 1loss/SRqqt1/detection_loss‚2.6898187536459703~ªô»½ôÖ /loss/FBt1/detection_loss‚2.4742157141367596 trainer/global_step‚5449 0loss/FBt4/assignment_loss‚0.7334896326065063 0loss/FRt2/assignment_loss‚3.7826568165490793 1loss/SRqqt1/detection_loss‚2.6898187536459703 % _timestamp‚1.7884492109899628e+09 _runtime‚4326.461484534 0loss/FRt4/assignment_loss‚2.9342671940724054 1loss/FBt3/assignment_loss‚0.21402082893267116 .loss/FRt3/detection_loss‚1.648828273303196 2loss/SRqqt2/assignment_loss‚1.4075100870560044 0loss/FBt2/assignment_loss‚0.3057563235228632 1loss/SRqqt3/detection_loss‚1.0896484214191635 1loss/SRqqt3/assignment_loss‚1.000711322024775 /loss/FBt3/detection_loss‚0.6778740182791542 _step‚447 1loss/SRqqt1/assignment_loss‚2.306918515608861 0loss/SRqqt2/detection_loss‚1.815163232062174 epoch‚4 /loss/FRt2/detection_loss‚1.9706316091293512 1loss/SRqqt4/detection_loss‚1.6276666702968734 .loss/FBt2/detection_loss‚4.264806091785431 .loss/FBt4/detection_loss‚3.840959846973419 %loss/total_loss‚48.64891007459618 /loss/FRt1/assignment_loss‚4.369876098632813 0loss/FRt3/assignment_loss‚1.7686523570389043 1loss/SRqqt4/assignment_loss‚1.50962668[¹ìDŸ2382077 /loss/FBt1/assignment_loss‚0.455564135064166 /loss/FRt4/detection_loss‚2.2928861488898593 /loss/FRt1/detection_loss‚3.4673613011837006¿ý•–£’õ‚(:‡ ãæÔ涩ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.055' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.611 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ûMblö‚(:á ãæÔÚ˜«ÿ disk.sda1.in‚0 disk.rbd1.out‚ 22.3271484375 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875) proc.memory.percent‚16.13900661468506 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.sda3.out‚ 168.30859375 disk.drbd2266.out‚0# proc.memory.rssMB‚ 2644.21484375 proc.cpu.threads‚17$ disk./.usageGB‚300.8865661621094 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚17447.5458984375 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.sda2.out‚0! proc.memory.availableMB‚950.5 cpu‚25.331529142929973 disk.drbd1716.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.dm-0.out‚17447.5458984375 disk.sda.in‚ 0.01171875# memory_percent‚94.1986083984375 network.sent‚ 1.2629477e+07 disk.dm-1.out‚ 168.30859375 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875) disk./.usagePercent‚20.19274991578376 disk.drbd2080.in‚0 disk.sda.out‚ 168.30859375 network.recv‚ 1.587031e+06 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚17447.5458984375 disk.drbd2266.in‚0^éü‘ž÷‚(:“ Ò£æÔÄ…üü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.837' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.367399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0·G¼mø‚(:â Ò£æÔÂîÃÿ& disk.nvme0c0n1.out‚17502.458984375 disk.sda1.out‚0( proc.memory.availableMB‚ 950.24609375# proc.memory.rssMB‚ 2644.21484375 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0! disk.dm-0.out‚17502.458984375 proc.cpu.threads‚17% disk./.usageGB‚300.89887619018555 disk.sda3.out‚ 168.953125 disk.nvme0n1.in‚0.078125) proc.memory.percent‚16.13900661468506 network.recv‚ 1.594504e+06 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 network.sent‚ 1.2738789e+07 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda1.in‚0$ disk.nvme0n1.out‚17502.458984375 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 168.953125* disk./.usagePercent‚20.193576052096756 disk.drbd2265.in‚0 disk.sda.out‚ 168.953125$ memory_percent‚94.20015811920166 cpu‚25.330505605350346 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 22.3271484375 disk.drbd1716.in‚0¶ &z2ù- _wandbruntime‚4362  _runtime‚4362v³ø¶ÇúÁ & lr-AdamW/pg1‚8.399897637038311e-05 & lr-AdamW/pg2‚8.399897637038311e-05 trainer/global_step‚5499 % _timestamp‚1.7884492467618964e+09 _runtime‚4362.233466208 _step‚448-¢P@ÌûÆ & lr-AdamW/pg1‚8.399897637038311e-05 & lr-AdamW/pg2‚8.399897637038311e-05 trainer/global_step‚5499 % _timestamp‚1.7884492467618964e+09 _runtime‚4362.233466208 _step‚448À*¥2ü- _wandbruntime‚4362  _runtime‚4362W^>à'ýÚO LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 103728024 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚323840717374976 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚459 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚353470971193344 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚310882445898752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚319239775404544 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚79818708 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚323840043009536 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚319239775404544 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚353470777337856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚323840717374976 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 103728024 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚319239793407488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚79818708 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4600923967488 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚72833363 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4826573886976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 183546734 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚79818710 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚72833824 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚315466433580064 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4583333243556$ƒX‡å'þßO GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4583333243556 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 103728024 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚323840717374976 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚459 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚353470971193344 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚310882445898752 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚319239775404544 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚79818708 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚323840043009536 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚319239775404544 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚353470777337856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚323840717374976 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 103728024 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚319239793407488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚79818708 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4600923967488 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚72833363 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4826573886976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 183546734 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚79818710 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚72833824 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚315466433580064 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756Á^çï2ÿ- _wandbruntime‚4362  _runtime‚4362ÞšNo!€åM >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4601087032832 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚46576878 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 183579230 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26269610 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚79832726 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚451 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚348706024103936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72846037 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚348706048044032 trainer/global_step‚5499 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 103746504 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚46576876 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚323897889854976 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72846488 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚315521482668472 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4584149678520 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚4362.918327872 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚323897889854976 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚79832724 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4827269322752 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚310937332989952 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚79832724 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4601087032832 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚46576878 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 183579230 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26269610 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚79832726 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚451 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚348706024103936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚72846037 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚348706048044032 trainer/global_step‚5499 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 103746504 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚46576876 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚323897889854976 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚72846488 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚315521482668472 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4584149678520 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚4362.918327872 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚323897889854976 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚79832724 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4827269322752 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚310937332989952 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚79832724 Eœyˆ‚(:î ð£æÔÀé¿€& disk.nvme0n1.out‚17614.40478515625 disk.drbd2080.in‚0# disk.dm-0.out‚17614.40478515625# proc.memory.rssMB‚ 2644.21484375 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125( disk.nvme0c0n1.out‚17614.40478515625 disk.sda3.out‚ 170.21484375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 170.21484375 disk.sda2.in‚0 disk.rbd1.out‚ 22.3896484375 proc.cpu.threads‚17 disk.rbd0.out‚0 disk.sda1.in‚0 disk.rbd1.in‚0 network.sent‚ 1.2853246e+07% disk./.usageGB‚300.91850662231445 network.recv‚ 1.605519e+06 disk.dm-1.out‚ 170.21484375 disk.drbd2265.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0$ memory_percent‚94.20034885406494( proc.memory.availableMB‚ 950.21484375* disk./.usagePercent‚20.194893466867956 disk.drbd1716.out‚0 disk.sda.in‚ 0.01171875) proc.memory.percent‚16.13900661468506 cpu‚25.328528702396618µ•¨zž‰‚(:“ ÿ£æÔüš—ý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚136.066' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.213199999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‘'œAcŠ‚(:Ø ÿ£æÔ®¤­€ network.sent‚ 1.2858293e+07* disk./.usagePercent‚20.195679409998075 disk.sda3.in‚ 0.01171875 disk.sda3.out‚170.6875# disk.dm-0.out‚17660.11181640625# proc.memory.rssMB‚ 2644.21484375 disk.drbd2080.in‚0% proc.memory.availableMB‚ 949.96875 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 network.recv‚ 1.608958e+06 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.drbd2265.in‚0 cpu‚25.32758629167732 disk.sda.out‚170.6875 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚17660.11181640625$ disk./.usageGB‚300.9302177429199 disk.drbd2080.out‚0 disk.dm-0.in‚0.078125$ memory_percent‚94.20185089111328 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0) proc.memory.percent‚16.13900661468506 disk.rbd0.in‚0 disk.drbd2265.out‚0 proc.cpu.threads‚17 disk.dm-1.out‚170.6875( disk.nvme0c0n1.out‚17660.11181640625 disk.sda1.in‚0 disk.rbd1.out‚22.53125 disk.drbd1716.out‚0=~ƃ2‹- _wandbruntime‚4398  _runtime‚4398)ÞEÜÇŒÁ & lr-AdamW/pg1‚8.313241487157057e-05 & lr-AdamW/pg2‚8.313241487157057e-05 trainer/global_step‚5549 % _timestamp‚1.7884492831897104e+09 _runtime‚4398.661397662 _step‚452ƒþ§ÌÆ & lr-AdamW/pg1‚8.313241487157057e-05 & lr-AdamW/pg2‚8.313241487157057e-05 trainer/global_step‚5549 % _timestamp‚1.7884492831897104e+09 _runtime‚4398.661397662 _step‚452Ä¡çm\2Ž- _wandbruntime‚4398  _runtime‚4398ëîµá'ÛO JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚313627652426752 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4641909773312 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚46977110 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚351726934189568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4869493693952 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4624168582964 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 104654076 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚356596587313152 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 104634038 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚322057445349888 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 104654076 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚351726910249472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚5549 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 185173586 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4869653123584 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚322057427346944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26493979 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 % _timestamp‚1.7884492831904776e+09 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚313627635387392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 _runtime‚4398.662367593 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚326698680757760 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚80519510 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4624823020720 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 _step‚453 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚73471087 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚326698680757760 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚73471552 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚80519510 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4641909773312 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚356596403943424 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚322057445349888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26494442 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚326699355123200 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚326698680757760 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚73471552 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚80519510 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 46419097733[hðœ12 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚356596403943424 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚322057445349888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26494442 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚326699355123200 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚313627652426752 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4641909773312 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚46977110 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚351726934189568 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4869493693952 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4624168582964 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 104654076 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚356596587313152 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 104634038 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚322057445349888 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 104654076 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚351726910249472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚5549 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 185173586 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414Åéœ?m2‘- _wandbruntime‚4399  _runtime‚4399à”äë&’åM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚46985278 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚322113798388736 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚80533524 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _step‚454 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚318308196013204 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 185186016 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚326756527603200 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚326756527603200 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚313682556557312 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚73484218 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26498940 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚313682539517952 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚80533526 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚322113780385792 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚356659101839360 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 104672556 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4642072838656 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 trainer/global_step‚5549 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚326755853224448 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚351788584927232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚322113798388736 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚322113780385792 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 104652492 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 104652492 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚480 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚318307524535880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884492838817372e+09 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73483736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚46985276 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚4399.353861379 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4642072838656 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 185206082 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 185186016 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4642729214464 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚80533524 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4642729214464 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 104672556 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚356658907997184 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4870347010048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚80533526 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26498460 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4625639455892 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4642729214464 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 104672556 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚356658907997184 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4870347010048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚80533526 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26498460 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4625639455892 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚80533524 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _step‚454 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚318308196013204 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 185186016 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚326756527603200 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚326756527603200 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚313682556557312 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚73484218 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26498940 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚313682539517952 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚80533526 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚322113780385792 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚356659101839360 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 104672556 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4642072838656 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 trainer/global_step‚5549 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚326755853224448 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚351788584927232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚322113798388736 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚322113780385792 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 104652492 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 104652492 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚480 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚318307524535880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884492838817372e+09 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚73483736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚46985276 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚4399.353861379 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4642072838656 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 185206082 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944Æu”²2”- _wandbruntime‚4399  _runtime‚439986èX•Ò 0loss/FRt1/assignment_loss‚4.4808526663553145 2loss/SRqqt1/assignment_loss‚2.2532064132392406 0loss/SRqqt4/assignment_loss‚1.32487083055079 1loss/SRqqt4/detection_loss‚1.8556845375150441 /loss/FBt2/assignment_loss‚0.414353281223027 /loss/FRt4/detection_loss‚2.5107447613890352 1loss/SRqqt2/detection_loss‚1.7014702553631829 _runtime‚4399.354879401 2loss/SRqqt3/assignment_loss‚1.0058081672196426 .loss/FBt4/detection_loss‚4.092435508966446 trainer/global_step‚5549 1loss/FBt4/assignment_loss‚0.09302244333321141 /loss/FRt2/detection_loss‚2.0141132804480466 /loss/FRt3/detection_loss‚1.7785992276571367 /loss/FBt3/detection_loss‚0.6979632775485516 1loss/FBt3/assignment_loss‚0.13907094321411895 epoch‚4 $ _timestamp‚1.788449283883333e+09 %loss/total_loss‚48.37573538808751 0loss/FRt4/assignment_loss‚3.6098634631205826 2loss/SRqqt2/assignment_loss‚1.7810729649528616 0loss/SRqqt1/detection_loss‚3.401875989511609 1loss/SRqqt3/detection_loss‚1.1145008350572279 ,loss/FBt1/detection_loss‚2.8675784712488 _step‚455 0loss/FRt2/assignment_loss‚3.7622873241251167 1loss/FBt1/assignment_loss‚0.43777252597619115 .loss/FRt1/detection_loss‚3.456268543288821 /loss/FBt2/detection_loss‚2.1801925961460387 0loss/FRt3/assignment_loss‚1.4021270806374757;¨a]–× 0loss/FRt1/assignment_loss‚4.4808526663553145 2loss/SRqqt1/assignment_loss‚2.2532064132392406 0loss/SRqqt4/assignment_loss‚1.32487083055079 1loss/SRqqt4/detection_loss‚1.8556845375150441 /loss/FBt2/assignment_loss‚0.414353281223027 /loss/FRt4/detection_loss‚2.5107447613890352 1loss/SRqqt2/detection_loss‚1.7014702553631829 _runtime‚4399.354879401 2loss/SRqqt3/assignment_loss‚1.0058081672196426 .loss/FBt4/detection_loss‚4.092435508966446 trainer/global_step‚5549 1loss/FBt4/assignment_loss‚0.09302244333321141 /loss/FRt2/detection_loss‚2.0141132804480466 /loss/FRt3/detection_loss‚1.7785992276571367 /loss/FBt3/detection_loss‚0.6979632775485516 1loss/FBt3/assignment_loss‚0.13907094321411895 epoch‚4 $ _timestamp‚1.788449283883333e+09 %loss/total_loss‚48.37573538808751 0loss/FRt4/assignment_loss‚3.6098634631205826 2loss/SRqqt2/assignment_loss‚1.7810729649528616 0loss/SRqqt1/detection_loss‚3.401875989511609 1loss/SRqqt3/detection_loss‚1.1145008350572279 ,loss/FBt1/detection_loss‚2.8675784712488 _step‚455 0loss/FRt2/assignment_loss‚3.7622873241251167 1loss/FBt1/assignment_loss‚0.43777252597619115 .loss/FRt1/detection_loss‚3.456268543288821 /loss/FBt2/detection_loss‚2.1801925961460387 0loss/FRt3/assignment_loss‚1.4021270806374757Çgj\“—‚(:ˆ Ž¤æÔì£ý gpu.0.gpu‚42.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.412' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8824 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0…“ q˜‚(:æ Ž¤æÔìÍÉÿ disk.nvme0c0n1.in‚0.078125 network.recv‚ 1.614017e+06 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚17712.26025390625 disk.dm-1.out‚ 171.28515625 disk.drbd1716.in‚0) proc.memory.percent‚16.13900661468506 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 171.28515625 proc.cpu.threads‚17 disk.sda1.out‚0 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875 cpu‚25.32665733775173 network.sent‚ 1.2865041e+07) disk./.usagePercent‚20.19643207214874 disk.rbd1.out‚22.53125( disk.nvme0c0n1.out‚17712.26025390625 disk.drbd2266.in‚0# disk.dm-0.out‚17712.26025390625% disk./.usageGB‚300.94143295288086 disk.drbd2080.out‚0# memory_percent‚94.1990613937378 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125( proc.memory.availableMB‚ 950.42578125 disk.sda2.out‚0 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.sda.out‚ 171.28515625# proc.memory.rssMB‚ 2644.21484375¸¾’™‚(:‡ ¤æÔ¬šý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚135.535' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚27.107 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ßC…'š‚(:ç ¤æÔ¾ÿ¸ÿ disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0# proc.memory.rssMB‚ 2644.21484375 disk.rbd0.out‚0# disk.dm-0.out‚17756.97119140625 disk.sda.in‚ 0.01171875 disk.rbd1.out‚22.59375 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 171.76953125( proc.memory.availableMB‚ 950.42578125) proc.memory.percent‚16.13900661468506$ disk./.usageGB‚301.2088050842285 disk.sda.out‚ 171.76953125 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚17756.97119140625 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2266.out‚0 cpu‚25.325727364867948 proc.cpu.threads‚17& disk.nvme0n1.out‚17756.97119140625 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 network.recv‚ 1.>ØàK.618749e+06* disk./.usagePercent‚20.214375640223643 disk.rbd0.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.sda3.out‚ 171.76953125# memory_percent‚94.1990613937378 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 network.sent‚ 1.2971767e+07Öˆ2›- _wandbruntime‚4435  _runtime‚4435»÷J×ÆœÀ _step‚456 & lr-AdamW/pg1‚8.226172925634519e-05 & lr-AdamW/pg2‚8.226172925634519e-05 trainer/global_step‚5599 $ _timestamp‚1.788449319692576e+09 _runtime‚4435.164192251lïúËÅ & lr-AdamW/pg2‚8.226172925634519e-05 trainer/global_step‚5599 $ _timestamp‚1.788449319692576e+09 _runtime‚4435.164192251 _step‚456 & lr-AdamW/pg1‚8.226172925634519e-05Ȉ‰}W2ž- _wandbruntime‚4435  _runtime‚4435{ÚKß'ŸÙO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 186780398 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 186800438 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4665003922372 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚329557992871424 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4682895579136 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚47385508 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚5599 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚316372841915392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚74108877 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26723369 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚321037845837764 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚324875097292288 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26723820 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _runtime‚ 4435.1650768 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚329557992871424 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 105580128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚329557318505984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚81220310 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚354809471072768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4912836025344 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 _step‚457 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4682239216640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚81220308 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚81220308 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4682895579136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚324875097292288 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚359722307098112 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚81220310 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 105560090 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4665658360128 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚359722119534080à8$×ä' ÞO JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚81220310 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 105560090 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4665658360128 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚359722119534080 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 186780398 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 186800438 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4665003922372 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚329557992871424 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4682895579136 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚47385508 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚5599 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚316372841915392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚74108877 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26723369 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚321037845837764 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚324875097292288 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26723820 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _runtime‚ 4435.1650768 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚329557992871424 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 105580128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚329557318505984 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚81220310 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚354809471072768 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4912836025344 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 _step‚457 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4682239216640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚81220308 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚81220308 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4682895579136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚324875097292288 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚359722307098112 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0É«GAž2¡- _wandbruntime‚4435  _runtime‚4435ÔDäíë&¢åM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚81234324 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚321093566403288 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4665820357336 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚47393678 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚324931450331136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26727866 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26728317 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 186812868 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚329615165351424 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 105598608 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74121542 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 186832934 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚81234324 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4913548582400 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4666474795300 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚329615165351424 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4683715020288 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4683715020288 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚324931432328192 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚359784646452736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚458 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 186832934 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 105578544 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 186812868 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚316427763085312 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚451 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 105598608 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚324931450331136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚47393676 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚321094237880612 trainer/global_step‚5599 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4683058644480 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚316427746045952 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚329614490972672 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚81234326 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 % _timestamp‚1.7884493203860066e+09 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚4435.858076819 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚81234326 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 105578544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚354871097870336 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 úiš£êM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚329615165351424 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 105598608 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74121542 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 186832934 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚81234324 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4913548582400 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4 cÐáV!666474795300 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚329615165351424 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4683715020288 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4683715020288 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚324931432328192 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚359784646452736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚458 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 186832934 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 105578544 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 186812868 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚316427763085312 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚451 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 105598608 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚324931450331136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚47393676 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚321094237880612 trainer/global_step‚5599 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4683058644480 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚316427746045952 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚329614490972672 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚81234326 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 % _timestamp‚1.7884493203860066e+09 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚4435.858076819 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚81234326 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 105578544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚354871097870336 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚81234324 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚321093566403288 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4665820357336 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚47393678 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚324931450331136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚26727866 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚26728317 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 186812868 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0Ê7ÞêA2¤- _wandbruntime‚4435  _runtime‚4435ÿh-ŽX¥Ò 2loss/SRqqt1/assignment_loss‚1.8193518256147703 2loss/SRqqt2/assignment_loss‚1.5521439127265835 /loss/FBt4/detection_loss‚3.8724149465560913 _runtime‚4435.859117483 0loss/FRt3/assignment_loss‚1.6310722430547078 .loss/FBt3/detection_loss‚0.758532120549882 .loss/FBt1/detection_loss‚2.691099241375923 /loss/FRt1/assignment_loss‚4.116002307488368 /loss/FRt3/detection_loss‚1.9582762751905691 1loss/SRqqt2/detection_loss‚2.0429350922887144 0loss/FBt1/assignment_loss‚0.6984775812083891 1loss/FBt4/assignment_loss‚0.16675275208156867 /loss/FRt4/detection_loss‚2.1245278964440026 /loss/SRqqt3/detection_loss‚1.06919098328799 2loss/SRqqt4/assignment_loss‚1.3707680067015906 .loss/FRt1/detection_loss‚3.167715930021726 1loss/SRqqt4/detection_loss‚1.8828670998751107 $loss/total_loss‚45.4082577995206 trainer/global_step‚5599 $ _timestamp‚1.788449320387567e+09 _step‚459 epoch‚4 0loss/FRt2/assignment_loss‚3.2598920452351474 0loss/FRt4/assignment_loss‚2.6777450177404614 2loss/SRqqt3/assignment_loss‚1.1025660764115552 0loss/FBt3/assignment_loss‚0.1750395892533872 /loss/FRt2/detection_loss‚2.0998724516557186 1loss/FBt2/assignment_loss‚0.35794106936139786 1loss/SRqqt1/detection_loss‚2.6115246673425037 .loss/FBt2/detection_loss‚2.201548668054434õŸ)"]¦× 2loss/SRqqt4/assignment_loss‚1.3707680067015906 .loss/FRt1/detection_loss‚3.167715930021726 1loss/SRqqt4/detection_loss‚1.8828670998751107 $loss/total_loss‚45.4082577995206 trainer/global_step‚5599 $ _timestamp‚1.788449320387567e+09 _step‚459 epoch‚4 0loss/FRt2/assignment_loss‚3.2598920452351474 0loss/FRt4/assignment_loss‚2.6777450177404614 2loss/SRqqt3/assignment_loss‚1.1025660764115552 0loss/FBt3/assignment_loss‚0.1750395892533872 /loss/FRt2/detection_loss‚2.0998724516557186 1loss/FBt2/assignment_loss‚0.35794106936139786 1loss/SRqqt1/detection_loss‚2.6115246673425037 .loss/FBt2/detection_loss‚2.201548668054434 2loss/SRqqt1/assignment_loss‚1.8193518256147703 2loss/SRqqt2/assignment_loss‚1.5521439127265835 /loss/FBt4/detection_loss‚3.8724149465560913 _runtime‚4435.859117483 0loss/FRt3/assignment_loss‚1.6310722430547078 .loss/FBt3/detection_loss‚0.758532120549882 .loss/FBt1/detection_loss‚2.691099241375923 /loss/FRt1/assignment_loss‚4.116002307488368 /loss/FRt3/detection_loss‚1.9582762751905691 1loss/SRqqt2/detection_loss‚2.0429350922887144 0loss/FBt1/assignment_loss‚0.6984775812083891 1loss/FBt4/assignment_loss‚0.16675275208156867 /loss/FRt4/detection_loss‚2.1245278964440026 /loss/SRqqt3/detection_loss‚1.06919098328799Ë«3“§‚(:ˆ ¬¤æÔÔײý gpu.0.gpu‚46.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.523' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.3046 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0TlÄTk¨‚(:à ¬¤æÔØÓ“€ disk.drbd2266.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 172.2734375# proc.memory.rssMB‚ 2644.21484375 cpu‚25.32479357060149 proc.cpu.threads‚17 network.sent‚ 1.2978297e+07 disk.sda.in‚ 0.01171875 disk.sda.out‚ 172.2734375# memory_percent‚94.2009449005127 disk.sda1.in‚0 disk.rbd1.out‚22.59375% disk.nvme0n1.out‚17799.1123046875 disk.dm-1.out‚ 172.2734375 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125) proc.memory.percent‚16.13900661468506 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.sda3.in‚ 0.01171875* disk./.usagePercent‚20.215207408702387% disk./.usageGB‚301.22119903564453 disk.rbd0.out‚0" disk.dm-0.out‚17799.1123046875 disk.sda1.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875' disk.nvme0c0n1.out‚17799.1123046875 disk.rbd0.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 network.recv‚ 1.623731e+06 disk.sda2.out‚0 disk.drbd2265.out‚0' proc.memory.availableMB‚ 950.1171875úø’>ž©‚(:“ »¤æÔÅ™—þ gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.937' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.187400000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¯=rª‚(:ç »¤æÔòœ­ÿ proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.sda1.in‚0 disk.sda1.out‚0" disk./.usageGB‚301.23193359375 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0 network.sent‚ 1.3086139e+07 network.recv‚ 1.629391e+06* disk./.usagePercent‚20.215927813903743 disk.nvme0c0n1.in‚0.078125" disk.dm-0.out‚17841.0849609375 disk.sda2.in‚0 disk.rbd1.out‚ 22.7353515625$ memory_percent‚94.19944286346436 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚17841.0849609375 disk.sda3.out‚ 172.84765625# proc.memory.rssMB‚ 2644.21484375) proc.memory.percent‚16.13900661468506 disk.dm-1.out‚ 172.84765625 disk.sda.out‚ 172.84765625 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 cpu‚25.32388361812669% disk.nvme0n1.out‚17841.0849609375 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875( proc.memory.availableMB‚ 950.36328125½úœ¾ž«‚(:“ ʤæÔ³ï°ý gpu.0.gpu‚35.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.011' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.002199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0½?§q¬‚(:æ ʤæÔ¯†‰ÿ# disk.dm-0.out‚17893.48974609375 disk.sda.in‚ 0.01171875$ memory_percent‚94.19951438903809' proc.memory.availableMB‚ 950.3515625 disk.nvme0n1.in‚0.078125( disk.nvme0c0n1.out‚17893.48974609375 disk.sda3.out‚ 173.3359375 disk.sda1.in‚0 disk.dm-1.out‚ 173.3359375 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 cpu‚25.322918273114134 network.sent‚ 1.3092907e+07* disk./.usagePercent‚20.215967239064014 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0) proc.memory.percent‚16.13903045654297$ disk./.usageGB‚301.2325210571289 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.rbd0.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 proc.memory.rssMB‚ 2644.21875 network.recv‚ 1.634651e+06 disk.rbd1.out‚ 22.7353515625 disk.sda.out‚ 173.3359375 disk.sda2.in‚0 proc.cpu.threads‚17 disk.rbd1.in‚0& disk.nvme0n1.out‚17893.48974609375 disk.sda3.in‚ 0.01171875 disk.sda2.out‚0År@P2­- _wandbruntime‚4471  _runtime‚4471Êì¡èÆ®À $ _timestamp‚1.788449356506264e+09 _runtime‚4471.977866354 _step‚460 & lr-AdamW/pg1‚8.138710097428418e-05 & lr-AdamW/pg2‚8.138710097428418e-05 trainer/global_step‚5649™,s[˯Š& lr-AdamW/pg1‚8.138710097428418e-05 & lr-AdamW/pg2‚8.138710097428418e-05 trainer/global_step‚5649 $ _timestamp‚1.788449356506264e+09 _runtime‚4471.977866354 _step‚460Ì:Z³¶2°- _wandbruntime‚4471  _runtime‚4471À„D°á'±ÛO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚332415956254208 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚74746402 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚26952492 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884493565069742e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚327692749234688 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚327692731231744 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 106486142 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚323823887705172 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚81921108 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚81921110 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4723881384960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚357892007955968 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚332416630619648 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚4471.978815195 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚362847998595584 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚332416630619648 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚323824559182288 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4723881384960 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4723225022464 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚47793910 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4705839261780 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚357891984015872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚461 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚332415956254208 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚362847811031552 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 188407250 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 106506180 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚438 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 106506180 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚319118065482752 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚319118048443392 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4955990639616 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚327692731231744 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚327692749234688 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚81921110 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26952054·±ô`²àO =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4723881384960 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4723225022464 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚47793910 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4705839261780 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚357891984015872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚461 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚332415956254208 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚362847811031552 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 188407250 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 106506180 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚438 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 106506180 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚319118065482752 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚319118048443392 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4955990639616 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚327692731231744 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚327692749234688 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚81921110 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚26952054 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚332415956254208 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚74746402 XKDeviceStatsMonit u†or.on_train_batch_start/inactive_split.small_pool.allocated‚26952492 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884493565069742e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚327692749234688 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚327692731231744 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 106486142 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚323823887705172 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚81921108 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚81921110 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4723881384960 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚357892007955968 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚332416630619648 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚4471.978815195 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚362847998595584 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚332416630619648 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚323824559182288 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 J1ŠÊ2³- _wandbruntime‚4472  _runtime‚4472”‚G®ë&´åM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚74758994 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4724044450304 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4724700826112 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 188459786 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚327749084270592 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 106504596 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚362910304506368 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚81935124 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚319172969613312 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 106504596 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4724044450304 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚47802078 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚332473803099648 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚81935126 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4472.655327253 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 188459786 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚362910502542848 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚327749102273536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74758540 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚47802076 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4956669752832 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884493571831412e+09 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4724700826112 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 188439720 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚81935124 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚47802078 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚332473803099648 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚81935126 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚4472.655327253 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 188459786 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚362910502542848 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚327749102273536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚74758540 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚47802076 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4956669752832 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884493571831412e+09 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4724700826112 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 188439720 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚81935124 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚74758994 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4724044450304 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4724700826112 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 188459786 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚327749084270592 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 106504596 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚362910304506368 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0΢¨!2¶- _wandbruntime‚4472  _runtime‚4472·{§òW·Ñ .loss/FRt2/detection_loss‚2.054415452739467 2loss/SRqqt4/assignment_loss‚1.2731800441492775 0loss/FBt1/assignment_loss‚0.5748359556267436 1loss/FBt4/assignment_loss‚0.28798850579187274 /loss/FBt2/detection_loss‚2.6538812555372715 /loss/FRt1/detection_loss‚2.8351129254987164 /loss/FRt1/assignment_loss‚4.791585414640365 /loss/FRt4/assignment_loss‚2.617458134281392 2loss/SRqqt2/assignment_loss‚1.5441617642206198 .loss/FRt3/detection_loss‚2.169264785835848 /loss/FBt1/detection_loss‚2.2532893071571984 0loss/FRt2/assignment_loss‚3.6678519793178723 /loss/FRt3/assignment_loss‚2.607814839073256 1loss/SRqqt3/detection_loss‚1.8208253764091653 -loss/FBt4/detection_loss‚3.64466243237257 %loss/total_loss‚48.11885906816089 trainer/global_step‚5649 % _timestamp‚1.7884493571847649e+09 _runtime‚4472.656290982 2loss/SRqqt1/assignment_loss‚1.6763596507635983 1loss/FBt3/assignment_loss‚0.14281701995011806 2loss/SRqqt3/assignment_loss‚1.5866014979621197 0loss/SRqqt4/detection_loss‚1.438346638786988 _step‚463 /loss/FBt3/detection_loss‚0.8856336913382014 epoch‚4 /loss/FRt4/detection_loss‚2.3252624702088687 0loss/SRqqt1/detection_loss‚2.669420018340602 0loss/FBt2/assignment_loss‚0.5711534062859744 0loss/SRqqt2/detection_loss‚2.026936501872783Ø5GÌ\¸Ö /loss/FBt3/detection_loss‚0.8856336913382014 epoch‚4 /loss/FRt4/detection_loss‚2.3252624702088687 0loss/SRqqt1/detection_loss‚2.669420018340602 0loss/FBt2/assignment_loss‚0.5711534062859744 0loss/SRqqt2/detection_loss‚2.026936501872783 .loss/FRt2/detection_loss‚2.054415452739467 2loss/SRqqt4/assignment_loss‚1.2731800441492775 0loss/FBt1/assignment_loss‚0.5748359556267436 1loss/FBt4/assignment_loss‚0.28798850579187274 /loss/FBt2/detection_loss‚2.6538812555372715 /loss/FRt1/detection_loss‚2.8351129254987164 /loss/FRt1/assignment_loss‚4.791585414640365 /loss/FRt4/assignment_loss‚2.617458134281392 2loss/SRqqt2/assignment_loss‚1.5441617642206198 .loss/FRt3/detection_loss‚2.169264785835848 /loss/FBt1/detection_loss‚2.2532893071571984 0loss/FRt2/assignment_loss‚3.6678519793178723 /loss/FRt3/assignment_loss‚2.607814839073256 1loss/SRqqt3/detection_loss‚1.8208253764091653 -loss/FBt4/detection_loss‚3.64466243237257 %loss/total_loss‚48.11885906816089 trainer/global_step‚5649 % _timestamp‚1.7884493571847649e+09 _runtime‚4472.656290982 2loss/SRqqt1/assignment_loss‚1.6763596507635983 1loss/FBt3/assignment_loss‚0.14281701995011806 2loss/SRqqt3/assignment_loss‚1.5866014979621197 0loss/SRqqt4/detection_loss‚1.438346638786988 _step‚463ϲµV£“¹‚(:ˆ Ù¤æÔÜüüü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.868' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9736 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0G¦‹vuº‚(:ê Ù¤æÔ“«‡ÿ disk.sda.out‚ 173.82421875 disk.rbd1.out‚ 22.7978515625 disk.drbd1716.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚301.2385482788086 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 173.82421875 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 network.recv‚ 1.638308e+06 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda1.out‚0& disk.nvme0n1.out‚17931.28271484375$ memory_percent‚94.20216083526611 proc.memory.rssMB‚ 2644.21875# disk.dm-0.out‚17931.28271484375 network.sent‚ 1.3098129e+07 disk.sda2.out‚0 disk.dm-1.out‚ 173.82421875 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.sda2.in‚0( proc.memory.availableMB‚ 949.91796875 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.dm-1.in‚ 0.01171875( disk.nvme0c0n1.out‚17931.28271484375 disk.sda.in‚ 0.01171875* disk./.usagePercent‚20.216371730968113 disk.drbd2266.in‚0) proc.memory.percent‚16.13903045654297 cpu‚25.322011612719358òÂ@æ“»‚(:ˆ è¤æÔßÄóü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.951' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.7902 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡ékr¼‚(:ç è¤æÔËÅ€$ disk./.usageGB‚301.2402801513672 disk.drbd2080.out‚0" disk.dm-0.out‚17978.8720703125 disk.sda1.out‚0 network.sent‚ 1.3207135e+07 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚17978.8720703125 disk.sda3.in‚ 0.01171875 disk.dm-1.out‚ 174.46484375 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0$ memory_percent‚94.20373439788818 disk.drbd2265.in‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.rbd1.out‚ 22.8837890625 disk.drbd2266.in‚0 disk.drbd2266.out‚0 network.recv‚ 1.645388e+06 disk.drbd2080.in‚0 proc.memory.rssMB‚ 2644.21875) proc.memory.percent‚16.13903045654297 disk.sda.in‚ 0.01171875 disk.sda.out‚ 174.46484375 disk.rbd1.in‚0 cpu‚25.321098349298875 disk.sda3.out‚ 174.46484375% disk.nvme0n1.out‚17978.8720703125( proc.memory.availableMB‚ 949.66015625* disk./.usagePercent‚20.216487958388658€åÓy2½- _wandbruntime‚4508  _runtime‚4508ÿߣǾÁ & lr-AdamW/pg1‚8.050871229661057e-05 & lr-AdamW/pg2‚8.050871229661057e-05 trainer/global_step‚5699 % _timestamp‚1.7884493929276657e+09 _runtime‚4508.399249962 _step‚464€1â†Ì¿Æ trainer/global_step‚5699 % _timestamp‚1.7884493929276657e+09 _runtime‚4508.399249962 _step‚464 & lr-AdamW/pg1‚8.050871229661057e-05 & lr-AdamW/pg2‚8.050871229661057e-05ÐóBz2À- _wandbruntime‚4508  _runtime‚4508ãïõìÁÚO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚48202310 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚335274594002432 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚5699 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 107432232 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚335274594002432 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚82621910 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚48202308 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚75384860 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 $ _timestamp‚1.788449392928364e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚75384397 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27182089 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚360974520899072 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4764867190784 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚335275268367872 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 190054142 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚82621908 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 _runtime‚4508.400199393 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4764210828288 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚330510383174144 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚365973541825536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4746674601188 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27182550 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚461 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4999020926464 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚330510401177088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4764867190784 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 107412194 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚463 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚335274594002432 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚5699 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 107432232 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚335274594002432 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚82621910 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚48202308 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚75384860 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 $ _timestamp‚1.788449392928364e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚75384397 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27182089 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚360974520899072 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4764867190784 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚335275268367872 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 190054142 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚82621908 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088ÑWt»"2Ã- _wandbruntime‚4509  _runtime‚4509!’§’ë&ÄåM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚82635924 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 190086638 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 107450712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚466 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 190066572 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚321918159101952 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚487 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚330566754215936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚82635926 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚82635924 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚75397178 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4747491036152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5000044095488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚5699 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 190066572 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 107430648 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 107450712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚326666321615428 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚335331766469120 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4748145474116 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚485 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚361036171636736 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚330566736212992 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 190086638 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4765686631936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4765030256128 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 190086638 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 107450712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚466 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 190066572 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚321918159101952 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚487 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚330566754215936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚82635926 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚82635924 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚75397178 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4747491036152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5000044095488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚5699 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 190066572 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 107430648 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 107450712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚326666321615428 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚335331766469120 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4748145474116 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚485 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚361036171636736 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚330566736212992 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 190086638 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4765686631936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4765030256128 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚48610710 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚338133231750656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4805852996608 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚473 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 108338246 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 191680994 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚333328035116544 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4805196634112 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 108358284 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚369099355472896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 _step‚469 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚329396642917104 % _timestamp‚1.7884494295165932e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4805852996608 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4805196634112 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27412650 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚83322708 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 206438400 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚4544.988420521 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚338133231750656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚48610708 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5042273750528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚364057057782272 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚324608461499392 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4787509940596 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚5749 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚76023360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5042091252224 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 191660954 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 108358284†Éæ'ÒàO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚333328035116544 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4805196634112 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 108358284 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚369099355472896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 _step‚469 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚329396642917104 % _timestamp‚1.7884494295165932e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4805852996608 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4805196634112 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27412650 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚83322708 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 206438400 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚4544.988420521 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚338133231750656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚48610708 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5042273750528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚364057057782272 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚324608461499392 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4787509940596 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚5749 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚76023360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5042091252224 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 191660954 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 108358284 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚364057081722368 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚83322708 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 182498304 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚324608478538752 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚338133906116096 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚333328053119488 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚48610710 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚338133231750656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4805852996608 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚473 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 108338246 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 191680994 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424Õšl ÑÓ‚(:† •¥æÔ×öûý gpu.0.gpu‚56.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚185.48' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚–lYo‚37.096 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÆTÓoÔ‚(:ä •¥æÔöÿ disk.dm-1.out‚ 175.92578125 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚18123.7861328125 disk.sda3.out‚ 175.92578125 disk.rbd0.out‚0 disk.dm-0.in‚0.078125% disk.nvme0n1.out‚18123.7861328125 cpu‚25.31844277857222 proc.cpu.threads‚17 network.recv‚ 1.659586e+06 disk.rbd0.in‚0 disk.sda1.out‚0 disk.drbd1716.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0$ memory_percent‚94.20382976531982$ disk./.usageGB‚301.2976493835449 disk.sda.out‚ 175.92578125 disk.drbd2080.out‚0 disk.sda2.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda2.out‚0 disk.rbd1.in‚0( disk./.usagePercent‚20.2203380556961 disk.sda1.in‚0 disk.rbd1.out‚ 23.2041015625 disk.dm-1.in‚ 0.01171875( proc.memory.availableMB‚ 949.64453125 proc.memory.rssMB‚ 2644.21875) proc.memory.percent‚16.13903045654297" disk.dm-0.out‚18123.7861328125 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0 network.sent‚ 1.3326715e+07 disk.sda.in‚ 0.01171875½ö#ƒ2Õ- _wandbruntime‚4545  _runtime‚4545êDJë&ÖåM OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 108356700 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚324663382669312 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚333384406158336 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚364118732460032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚76035570 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚338190404217344 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 108376764 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚76036021 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚329451692005512 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 108356700 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4788980813524 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 191693424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚5749 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚333384406158336 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5042941684736 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 108376764 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4806016061952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚324663365629952 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4806672437760 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5043113683968 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27417143 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚333384388155392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚451 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚364118708519936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 191713490 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚83336726 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚338191078596096 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 191713490 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27416694 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚338191078596096 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884494301970615e+09 _step‚470 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 191693424 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚83336724 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 171999232 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚369161650204672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4806672437760 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚83336726 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4788326375560 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚333384388155392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚369161846144000 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚329452363482836 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚48618876 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚76036021 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚329451692005512 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 108356700 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4788980813524 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 191693424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚5749 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚333384406158336 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5042941684736 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 108376764 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4806016061952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚324663365629952 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4806672437760 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5043113683968 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27417143 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚333384388155392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚451 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚364118708519936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 191713490 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚83336726 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚338191078596096 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 191713490 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27416694 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚338191078596096 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 % _timestamp‚1.7884494301970615e+09 _step‚470 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 191693424 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚83336724 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 171999232 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚369161650204672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4806672437760 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚83336726 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 336208384 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4788326375560 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚333384388155392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚369161846144000 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚329452363482836 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚48618876 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚333384406158336 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚364118732460032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚76035570 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0Ö?oý2Ø- _wandbruntime‚4545  _runtime‚4545Çà_‡TÙÎ % _timestamp‚1.7884494301985693e+09 /loss/FRt4/assignment_loss‚2.808782491900704 2loss/SRqqt3/assignment_loss‚1.0783802524829904 2loss/SRqqt4/assignment_loss‚1.7595524632185697 /loss/FBt2/assignment_loss‚0.865571595329117 .loss/FRt1/detection_loss‚2.917966978890555 0loss/SRqqt1/detection_loss‚2.259260852547253 /loss/FBt2/detection_loss‚2.0963278902428493 epoch‚4 2loss/SRqqt2/assignment_loss‚1.3839397768356971 1loss/FBt3/assignment_loss‚0.18821691566630744 .loss/FBt1/detection_loss‚2.770821257432302 /loss/FBt3/detection_loss‚0.7883504026879867 _runtime‚4545.670082879 /loss/FRt3/assignment_loss‚2.051798678257249 1loss/SRqqt2/detection_loss‚1.8281791394665128 /loss/FRt2/assignment_loss‚3.279533329162192 %loss/total_loss‚46.82633004927743 0loss/FRt1/assignment_loss‚4.7366607785224915 2loss/SRqqt1/assignment_loss‚2.2900355914059807 0loss/FBt4/assignment_loss‚0.3112306477058155 -loss/FRt2/detection_loss‚2.19844097184374 /loss/FRt3/detection_loss‚1.8925698207352648 1loss/SRqqt4/detection_loss‚1.7441243714094161 trainer/global_step‚5749 _step‚471 .loss/FBt1/assignment_loss‚0.42408335039745 /loss/FRt4/detection_loss‚2.1962401287122204 1loss/SRqqt3/detection_loss‚1.3182589197531343 .loss/FBt4/detection_loss‚3.638003444671631¹C·ÖYÚÓ .loss/FBt4/detection_loss‚3.638003444671631 % _timestamp‚1.7884494301985693e+09 /loss/FRt4/assignment_loss‚2.808782491900704 2loss/SRqqt3/assignment_loss‚1.0783802524829904 2loss/SRqqt4/assignment_loss‚1.7595524632185697 /loss/FBt2/assignment_loss‚0.865571595329117 .loss/FRt1/detection_loss‚2.917966978890555 0loss/SRqqt1/detection_loss‚2.259260852547253 /loss/FBt2/detection_loss‚2.0963278902428493 epoch‚4 2loss/SRqqt2/assignment_loss‚1.3839397768356971 1loss/FBt3/assignment_loss‚0.18821691566630744 .loss/FBt1/detection_loss‚2.770821257432302 /loss/FBt3/detection_loss‚0.7883504026879867 _runtime‚4545.670082879 /loss/FRt3/assignment_loss‚2.051798678257249 1loss/SRqqt2/detection_loss‚1.8281791394665128 /loss/FRt2/assignment_loss‚3.279533329162192 %loss/total_loss‚46.82633004927743 0loss/FRt1/assignment_loss‚4.7366607785224915 2loss/SRqqt1/assignment_loss‚2.2900355914059807 0loss/FBt4/assignment_loss‚0.3112306477058155 -loss/FRt2/detection_loss‚2.19844097184374 /loss/FRt3/detection_loss‚1.8925698207352648 1loss/SRqqt4/detection_loss‚1.7441243714094161 trainer/global_step‚5749 _step‚471 .loss/FBt1/assignment_loss‚0.42408335039745 /loss/FRt4/detection_loss‚2.1962401287122204 1loss/SRqqt3/detection_loss‚1.3182589197531343×# ÚxžÛ‚(:“ ¤¥æÔ ˜äý gpu.0.gpu‚47.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.123' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.624599999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0JÄïsÜ‚(:è ¤¥æÔ´®‡€# disk.dm-0.out‚18177.27294921875 disk.sda.out‚ 176.54296875 disk.drbd1716.in‚0 disk.drbd2265.out‚0# memory_percent‚94.2039966583252) proc.memory.percent‚16.13903045654297 disk.sda1.out‚0& disk.nvme0n1.out‚18177.27294921875 disk.sda3.out‚ 176.54296875 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd1.in‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875' proc.memory.availableMB‚ 949.6171875 network.recv‚ 1.666454e+06 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.dm-1.out‚ 176.54296875 disk.drbd2080.out‚0 cpu‚25.317500322061704 network.sent‚ 1.3424451e+07 disk.dm-0.in‚0.078125 disk.drbd1716.out‚0* disk./.usagePercent‚20.221360549787853$ disk./.usageGB‚301.3128852844238 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.rbd1.out‚ 23.3759765625( disk.nvme0c0n1.out‚18177.27294921875 proc.memory.rssMB‚ 2644.21875 fh“Ý‚(:ˆ ³¥æÔÞê„ý gpu.0.gpu‚26.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚147.292' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚29.4584 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Üj6¡gÞ‚(:Ü ³¥æÔðõ¥€ disk.dm-1.out‚ 177.1328125 network.sent‚ 1.3521602e+07! disk./.usageGB‚301.3193359375 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.drbd2080.out‚0) proc.memory.percent‚16.13903045654297 cpu‚25.316626110513255 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2265.in‚0 disk.sda.in‚ 0.01171875 network.recv‚ 1.671851e+06! disk.dm-0.out‚18217.189453125& proc.memory.availableMB‚ 949.859375 disk.rbd0.in‚0$ memory_percent‚94.20251846313477 disk.sda3.out‚ 177.1328125 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0& disk.nvme0c0n1.out‚18217.189453125 proc.cpu.threads‚17* disk./.usagePercent‚20.221793458528253 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 23.5478515625 disk.drbd1716.in‚0 proc.memory.rssMB‚ 2644.21875 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0$ disk.nvme0n1.out‚18217.189453125 disk.sda.out‚ 177.1328125 disk.sda1.in‚0E®;52ß- _wandbruntime‚4581  _runtime‚4581$q!±ÇàÁ & lr-AdamW/pg1‚7.874138671947347e-05 & lr-AdamW/pg2‚7.874138671947347e-05 trainer/global_step‚5799 % _timestamp‚1.7884494658210351e+09 _runtime‚4581.292631047 _step‚472ÉŸ&ÖÌáÆ trainer/global_step‚5799 % _timestamp‚1.7884494658210351e+09 _runtime‚4581.292631047 _step‚472 & lr-AdamW/pg1‚7.874138671947347e-05 & lr-AdamW/pg2‚7.874138671947347e-05Ø8Æ92â- _wandbruntime‚4581  _runtime‚4581S9äªãÚO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 193307846 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _runtime‚ 4581.29350194 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚340992543864320 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5085431625728 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚336145705061888 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚340991869498880 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚332182684784512 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4846838802432 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚372224873153024 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚76660530 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚336145687058944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚49019110 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177078272 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4846182439936 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4828999717760 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 109284336 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚84023510 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 109284336 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚336145705061888 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚5799 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚367139594665472 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27641422 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚340991869498880 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚332182684784512 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4846838802432 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚372224873153024 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚76660530 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚336145687058944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚49019110 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177078272 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4846182439936 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4828999717760 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 109284336 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚84023510 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 109284336 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚336145705061888 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚5799 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚367139594665472 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27641422 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _runtime‚ 4581.29350194 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚340992543864320 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5085431625728 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚336145705061888 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚336202058100736 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚84037526 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚76673739 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚367201245403136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚459 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 193320276 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚84037526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚49027276 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 109282752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚461 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 109302816 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚341049716344320 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚341049041965568 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4847001867776 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚84037524 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27646002 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5086126999040 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚327408589197312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4829161714968 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 109282752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5086298998272 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚332238405350244 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚49027278 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚5799 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 193340342 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 193340342 _step‚474 % _timestamp‚1.7884494664954562e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚336202058100736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27646461 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚341049716344320 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4847658243584 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚336202040097792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 193320276 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚327408572157952 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 _runtime‚4581.967857904 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚84037524 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚76673278 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4829816152932 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872WüG¥çêM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚84037526 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚76673739 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚367201245403136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚459 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 193320276 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚84037526 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚49027276 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 109282752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚461 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 109302816 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚341049716344320 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚341049041965568 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4847001867776 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚84037524 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27646002 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5086126999040 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚327408589197312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4829161714968 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 109282752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5086298998272 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚332238405350244 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚49027278 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 trainer/global_step‚5799 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 193340342 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 193340342 _step‚474 % _timestamp‚1.7884494664954562e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚336202058100736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27646461 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚341049716344320 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4847658243584 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚336202040097792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 193320276 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚327408572157952 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 _runtime‚5ƒV³K 4581.967857904 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚84037524 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚76673278 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4829816152932 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚336202040097792 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚372287372402176 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚367201269343232 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚332237733872920 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚336202058100736 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430Úx à]2è- _wandbruntime‚4581  _runtime‚4581¢ìèmSéÍ epoch‚4 _step‚475 0loss/FRt1/assignment_loss‚4.6030117238269135 0loss/FBt2/assignment_loss‚0.3628251875439972 0loss/SRqqt4/detection_loss‚1.419732487103978 .loss/FBt2/detection_loss‚2.695215548078219 _runtime‚4581.969172815 /loss/FRt3/assignment_loss‚1.978165994087855 1loss/SRqqt1/detection_loss‚2.5113058704318423 1loss/SRqqt2/detection_loss‚1.9640112168022565 1loss/SRqqt3/detection_loss‚1.2387104555408457 /loss/FRt2/assignment_loss‚3.659921184499213 1loss/SRqqt1/assignment_loss‚2.746281439156243 1loss/SRqqt2/assignment_loss‚1.908155904195848 .loss/FRt3/detection_loss‚1.951550349238373 % _timestamp‚1.7884494664973295e+09 1loss/SRqqt3/assignment_loss‚1.514830518430516 2loss/SRqqt4/assignment_loss‚1.5189663424599367 .loss/FRt2/detection_loss‚2.188209171624894 trainer/global_step‚5799 /loss/FBt3/detection_loss‚0.7701240440458059 %loss/total_loss‚48.56295468908786 0loss/FBt3/assignment_loss‚0.2833834014347467 1loss/FBt4/assignment_loss‚0.18376352256079068 .loss/FRt1/detection_loss‚2.981193752849803 /loss/FRt4/detection_loss‚2.2941050204965805 /loss/FRt4/assignment_loss‚3.014628995789422 /loss/FBt1/assignment_loss‚0.592701495933928 /loss/FBt4/detection_loss‚3.2315542697906494 .loss/FBt1/detection_loss‚2.950606793165207lUŸ¨XêÒ /loss/FBt3/detection_loss‚0.7701240440458059 %loss/total_loss‚48.56295468908786 0loss/FBt3/assignment_loss‚0.2833834014347467 1loss/FBt4/assignment_loss‚0.18376352256079068 .loss/FRt1/detection_loss‚2.981193752849803 /loss/FRt4/detection_loss‚2.2941050204965805 /loss/FRt4/assignment_loss‚3.014628995789422 /loss/FBt1/assignment_loss‚0.592701495933928 /loss/FBt4/detection_loss‚3.2315542697906494 .loss/FBt1/detection_loss‚2.950606793165207 epoch‚4 _step‚475 0loss/FRt1/assignment_loss‚4.6030117238269135 0loss/FBt2/assignment_loss‚0.3628251875439972 0loss/SRqqt4/detection_loss‚1.419732487103978 .loss/FBt2/detection_loss‚2.695215548078219 _runtime‚4581.969172815 /loss/FRt3/assignment_loss‚1.978165994087855 1loss/SRqqt1/detection_loss‚2.5113058704318423 1loss/SRqqt2/detection_loss‚1.9640112168022565 1loss/SRqqt3/detection_loss‚1.2387104555408457 /loss/FRt2/assignment_loss‚3.659921184499213 1loss/SRqqt1/assignment_loss‚2.746281439156243 1loss/SRqqt2/assignment_loss‚1.908155904195848 .loss/FRt3/detection_loss‚1.951550349238373 % _timestamp‚1.7884494664973295e+09 1loss/SRqqt3/assignment_loss‚1.514830518430516 2loss/SRqqt4/assignment_loss‚1.5189663424599367 .loss/FRt2/detection_loss‚2.188209171624894 trainer/global_step‚5799ۙ˓ë‚(:ˆ Â¥æÔðƒ…ý gpu.0.gpu‚46.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚127.973' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.5946 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0{rÌêkì‚(:à Â¥æÔµµ÷ÿ disk.nvme0n1.in‚0.078125 disk.drbd2266.out‚0 network.recv‚ 1.677077e+06 disk.drbd1716.out‚0 disk.sda3.out‚ 177.95703125 disk.sda.out‚ 177.95703125) proc.memory.percent‚16.13903045654297$ disk./.usageGB‚301.3284378051758 disk.sda2.in‚0 disk.rbd1.out‚ 23.5478515625 disk.drbd2265.out‚0 disk.drbd2080.out‚0 cpu‚25.315710451191652$ disk.nvme0n1.out‚18260.814453125 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.drbd2266.in‚0) disk./.usagePercent‚20.22240429250495 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 177.95703125 disk.drbd1716.in‚0% proc.memory.availableMB‚ 949.53125 proc.memory.rssMB‚ 2644.21875 network.sent‚ 1.3528448e+07 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0! disk.dm-0.out‚18260.814453125 disk.sda1.out‚0 proc.cpu.threads‚17& disk.nvme0c0n1.out‚18260.814453125 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0$ memory_percent‚94.20452117919922\×\I“í‚(:ˆ Ñ¥æÔõýñü gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.442' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6884 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Á$uî‚(:ê Ñ¥æÔ¬Äûÿ disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0 disk.drbd2266.out‚0 disk.dm-1.out‚ 178.28515625 disk.drbd1716.in‚0 disk.drbd1716.out‚0$ memory_percent‚94.20454502105713% disk./.usageGB‚301.33438873291016 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚18307.28662109375 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚18307.28662109375 disk.dm-0.in‚0.078125 disk.dm-1.in‚ 0.01171875( proc.memory.availableMB‚ 949.52734375 network.recv‚ 1.681874e+06 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0) proc.memory.percent‚16.13903045654297 proc.cpu.threads‚17 network.sent‚ 1.363614e+07* disk./.usagePercent‚20.222803664258365 disk.sda3.out‚ 178.28515625 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 23.6103515625 disk.drbd2265.out‚0 proc.memory.rssMB‚ 2644.21875 disk.sda3.in‚ 0.01171875# disk.dm-0.out‚18307.28662109375 disk.sda.in‚ 0.01171875 cpu‚25.314796034312863 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 178.28515625 disk.sda1.out‚0ÛÖz2ï- _wandbruntime‚4617  _runtime‚4617:hÈÇðÁ trainer/global_step‚5849 % _timestamp‚1.7884495019813745e+09 _runtime‚4617.452983891 _step‚476 & lr-AdamW/pg1‚7.785281812801135e-05 & lr-AdamW/pg2‚7.785281812801135e-05æxŒÌñÆ _runtime‚4617.452983891 _step‚476 & lr-AdamW/pg1‚7.785281812801135e-05 & lr-AdamW/pg2‚7.785281812801135e-05 trainer/global_step‚5849 % _timestamp‚1.7884495019813745e+09Ü$þíœ2ò- _wandbruntime‚4617  _runtime‚4617+"éá'óÛO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚84724308 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 110190350 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚338963339001344 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 110210388 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4887824608256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚375350746508288 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 194914658 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27870748 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚334968726651920 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚330098891594752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚5849 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚77297813 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _step‚477 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚343850507247104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚84724310 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5128433687040 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚84724310 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚338963357004288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚370222131548672 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚49427510 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚4617.453892997 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚343850507247104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4869180619412 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚375350565235712 % _timestamp‚1.7884495019820943e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4887824608256 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚330098874555392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚343851181612544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚77298258 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 110190350 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚343851181612544 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27870305 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4869835057168 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚370222155488768 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚338963339001344 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4887168245760 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚49427508 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4887168245760 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 194934698 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 110210388 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚338963357004288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚84724308 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5128591019520 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0䔶ræ'ôàO `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚338963357004288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚370222131548672 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚49427510 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _runtime‚4617.453892997 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚343850507247104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4869180619412 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚375350565235712 % _timestamp‚1.7884495019820943e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4887824608256 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚330098874555392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚343851181612544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚77298258 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 110190350 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚343851181612544 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚27870305 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4869835057168 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 336208384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚370222155488768 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚338963339001344 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4887168245760 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚49427508 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4887168245760 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 194934698 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 110210388 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚338963357004288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚84724308 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5128591019520 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚84724308 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 110190350 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚338963339001344 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 110210388 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4887824608256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚375350746508288 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 194914658 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚27870748 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚334968726651920 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚330098891594752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚5849 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚77297813 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _step‚477 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚343850507247104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚84724310 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5128433687040 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚84724310 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0ÝoR§j2õ- _wandbruntime‚4618  _runtime‚4618ײ§r öåM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚84738326 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5129283354624 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚339019692040192 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 194967194 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 110208804 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 110208804 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚330153778685952 T@DeviceStatsMonitor.o­†yn_train_batch_end/active_bytes.all.allocated‚343908354092544 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4887987673600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5129451159552 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚335024447217652 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 110228868 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚339019692040192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4869997054376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4887987673600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 _step‚478 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚5849 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚339019710043136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚343908354092544 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 194947128 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚370283806226432 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚339019710043136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 _runtime‚4618.148482828 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 194947128 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 194967194 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27875334 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚84738324 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 110228868 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚77310558 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27874882 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚84738324 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4888644049408 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 % _timestamp‚1.7884495026762977e+09 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚77310558 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚27874882 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚84738324 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4888644049408 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 % _timestamp‚1.7884495026762977e+09 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5129283354624 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚339019692040192 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 194967194 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 110208804 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 110208804 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚330153778685952 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚343908354092544 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4887987673600 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5129451159552 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚335024447217652 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 110228868 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚339019692040192 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4869997054376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4887987673600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 _step‚478 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚5849 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚339019710043136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚343908354092544 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 194947128 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚370283806226432 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚339019710043136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 _runtime‚4618.148482828 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 194947128 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 194967194 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚27875334 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚84738324 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 110228868Þí¦ë2ø- _wandbruntime‚4618  _runtime‚4618;ðöˆYùÓ 2loss/SRqqt3/assignment_loss‚0.9024255295785574 0loss/FBt4/assignment_loss‚0.2181332656997256 /loss/FBt3/detection_loss‚0.7800888185050677 trainer/global_step‚5849 _step‚479 0loss/FRt4/assignment_loss‚3.4446091602245965 /loss/FBt4/detection_loss‚3.5860853254795075 % _timestamp‚1.7884495026781476e+09 0loss/FRt3/assignment_loss‚1.7442962300777436 2loss/SRqqt1/assignment_loss‚2.0214453977532685 2loss/SRqqt2/assignment_loss‚1.2705994241009906 2loss/SRqqt4/assignment_loss‚1.3989299800484738 $loss/total_loss‚46.2754145777184 epoch‚4 /loss/FRt1/assignment_loss‚5.103170677467629 /loss/FRt2/assignment_loss‚3.204899940574378 0loss/SRqqt1/detection_loss‚2.676530549302697 0loss/SRqqt4/detection_loss‚1.853431043314173 _runtime‚4618.149634883 /loss/FRt4/detection_loss‚2.2611916310464344 1loss/SRqqt3/detection_loss‚1.0553566982826361 .loss/FBt1/detection_loss‚3.021218756834666 /loss/FBt2/detection_loss‚2.1851630154110135 .loss/FRt1/detection_loss‚3.122952046217742 .loss/FRt2/detection_loss‚2.008292910299803 1loss/FBt1/assignment_loss‚0.33588384524950166 0loss/FBt2/assignment_loss‚0.5437355731568932 1loss/SRqqt2/detection_loss‚1.6192117260312133 1loss/FBt3/assignment_loss‚0.22002907097286598 /loss/FRt3/detection_loss‚1.6977339620888232r}â^úØ 0loss/FRt4/assignment_loss‚3.4446091602245965 /loss/FBt4/detection_loss‚3.5860853254795075 % _timestamp‚1.7884495026781476e+09 0loss/FRt3/assignment_loss‚1.7442962300777436 2loss/SRqqt1/assignment_loss‚2.0214453977532685 2loss/SRqqt2/assignment_loss‚1.2705994241009906 2loss/SRqqt4/assignment_loss‚1.3989299800484738 $loss/total_loss‚46.2754145777184 epoch‚4 /loss/FRt1/assignment_loss‚5.103170677467629 /loss/FRt2/assignment_loss‚3.204899940574378 0loss/SRqqt1/detection_loss‚2.676530549302697 0loss/SRqqt4/detection_loss‚1.853431043314173 _runtime‚4618.149634883 /loss/FRt4/detection_loss‚2.2611916310464344 1loss/SRqqt3/detection_loss‚1.0553566982826361 .loss/FBt1/detection_loss‚3.021218756834666 /loss/FBt2/detection_loss‚2.1851630154110135 .loss/FRt1/detection_loss‚3.122952046217742 .loss/FRt2/detection_loss‚2.008292910299803 1loss/FBt1/assignment_loss‚0.33588384524950166 0loss/FBt2/assignment_loss‚0.5437355731568932 1loss/SRqqt2/detection_loss‚1.6192117260312133 1loss/FBt3/assignment_loss‚0.22002907097286598 /loss/FRt3/detection_loss‚1.6977339620888232 2loss/SRqqt3/assignment_loss‚0.9024255295785574 0loss/FBt4/assignment_loss‚0.2181332656997256 /loss/FBt3/detection_loss‚0.7800888185050677 trainer/global_step‚5849 _step‚479ßG¬¹(“û‚(:ˆ à¥æÔƒî˜ý gpu.0.gpu‚46.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚173.309' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.6618 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0”éqyoü‚(:ä à¥æԬ˦ÿ% disk./.usageGB‚301.34361267089844( disk.nvme0c0n1.out‚18345.18115234375 disk.drbd2266.out‚0 disk.rbd0.in‚0$ memory_percent‚94.20456886291504 proc.memory.rssMB‚ 2644.21875 disk.dm-1.out‚ 178.9140625 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 178.9140625 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.sda2.out‚0' proc.memory.availableMB‚ 949.5234375 cpu‚25.31400279537872 disk.drbd2265.in‚0 disk.drbd2080.in‚0# disk.dm-0.out‚18345.18115234375 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 network.recv‚ 1.68662e+06 disk.sda.out‚ 178.9140625 disk.sda1.in‚0) disk./.usagePercent‚20.22342269047616 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.rbd1.out‚ 23.6103515625 disk.sda3.in‚ 0.01171875 disk.rbd1.in‚0 proc.cpu.threads‚17 network.sent‚ 1.3642605e+07 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚18345.18115234375) proc.memory.percent‚16.13903045654297ðÊ–â‘ý‚(:† ï¥æÔåžžý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚176.56' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.312 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0á’ßpþ‚(:å ï¥æÔŠ½ÿ disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 179.37109375 disk.drbd2266.in‚0 proc.cpu.threads‚17 disk.rbd1.out‚ 23.751953125 disk.drbd2266.out‚0 disk.rbd0.out‚0$ memory_percent‚94.20320987701416% disk./.usageGB‚301.34940338134766 disk.drbd2080.out‚0( proc.memory.availableMB‚ 949.74609375 proc.memory.rssMB‚ 2644.21875 network.sent‚ 1.3750258e+07 network.recv‚ 1.692066e+06) disk./.usagePercent‚20.22381130991314 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚18389.2001953125" disk.dm-0.out‚18389.2001953125 disk.sda.out‚ 179.37109375 disk.sda2.in‚0 disk.drbd2080.in‚0 cpu‚25.31309818984897 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.rbd0.in‚0) proc.memory.percent‚16.13903045654297 disk.sda1.out‚0' disk.nvme0c0n1.out‚18389.2001953125 disk.sda3.out‚ 179.37109375 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.drbd1716.out‚0ÈŒžÿ‚(:“ þ¥æÔàÕŽý gpu.0.gpu‚48.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.374' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.474799999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0fbtüq€‚(:æ þ¥æÔËõüþ network.recv‚ 1.697382e+06$ disk./.usageGB‚301.3600730895996 disk.sda2.in‚0 disk.sda3.out‚ 179.91796875) proc.memory.percent‚16.13903045654297 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚18415.0595703125 disk.sda3.in‚ 0.01171875 cpu‚25.312313926267684 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚18415.0595703125 disk.nvme0c0n1.in‚0.078125 proc.memory.rssMB‚ 2644.21875 disk.sda1.out‚0 disk.rbd1.out‚ 23.751953125 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0$ memory_percent‚94.20473575592041 proc.cpu.threads‚17 network.sent‚ 1.3756889e+07 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0( proc.memory.availableMB‚ 949.49609375 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.dm-1.out‚ 179.91796875 disk.drbd2265.in‚0" disk.dm-0.out‚18415.0595703125* disk./.usagePercent‚20.224527362986404 disk.sda.out‚ 179.91796875 disk.dm-1.in‚ 0.01171875 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 xN­2- _wandbruntime‚4654  _runtime‚4654'ÆäpÆ‚À % _timestamp‚1.7884495388723245e+09 _runtime‚ 4654.34400806 _step‚480 & lr-AdamW/pg1‚7.696122568018447e-05 & lr-AdamW/pg2‚7.696122568018447e-05 trainer/global_step‚5899ù¥™‹ËƒÅ & lr-AdamW/pg1‚7.696122568018447e-05 & lr-AdamW/pg2‚7.696122568018447e-05 trainer/global_step‚5899 % _timestamp‚1.7884495388723245e+09 _runtime‚ 4654.34400806 _step‚480à–áår2„- _wandbruntime‚4654  _runtime‚4654¯V#Ü7…ÛO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚373304668431872 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚341780990943744 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4928154051584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚346709819360768 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚341781008946688 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 111116402 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚85425110 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5171891089920 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 111136440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚346709144995328 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚341780990943744 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4928810414080 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚378476583461888 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 111116402 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _runtime‚4654.344917807 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5171725368832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4910015958820 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4910670396576 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚49835908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚378476393800704 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚341781008946688 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4928154051584 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batc€í59ªh_start/inactive_split_bytes.large_pool.allocated‚373304692371968 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 % _timestamp‚1.7884495388730476e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚85425108 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚346709144995328 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚332844081083392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚49835910 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28099869 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚332844098122752 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4928810414080 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28099414 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚77935322–Éb!æ'†àO @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚332844081083392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚49835910 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28099869 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚332844098122752 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4928810414080 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28099414 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚77935322 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚373304668431872 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚341780990943744 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4928154051584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚346709819360768 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚341781008946688 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 111116402 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚85425110 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5171891089920 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 111136440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚346709144995328 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚341780990943744 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4928810414080 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚378476583461888 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 111116402 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _runtime‚4654.344917807 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5171725368832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4910015958820 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4910670396576 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚49835908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚378476393800704 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚341781008946688 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4928154051584 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚373304692371968 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 % _timestamp‚1.7884495388730476e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚85425108 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚346709144995328á2*2‡- _wandbruntime‚4655  _runtime‚4655G¯+Që&ˆåM WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 196573980 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚49844078 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 trainer/global_step‚5899 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚373366319169536 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5172751934464 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚337809817607736 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚341837343982592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚341837343982592 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚378539095044096 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 111134856 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚85439124 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 196594046 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚373366343109632 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚85439126 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4928973479424 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚77948452 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚473 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 111134856 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28103901 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5172584129536 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚475 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚77947977 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚28104374 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 196594046 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 111154920 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚341837361985536 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4911486831748 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚4655.028968931 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4928973479424 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚341837361985536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚346766317462016 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4929629855232 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4929629855232 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚49844076 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752ÉhËð&‰êM D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚4655.028968931 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4928973479424 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚341837361985536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚346766317462016 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4929629855232 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4929629855232 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚49844076 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 196573980 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚49844078 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 trainer/global_step‚5899 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚373366319169536 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5172751934464 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚337809817607736 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚341837343982592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚341837343982592 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚378539095044096 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 111134856 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚85439124 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 196594046 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚373366343109632 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚85439126 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4928973479424 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚77948452 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚473 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 111134856 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28103901 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5172584129536 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚475 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚77947977 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚28104374 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 196594046 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 111154920 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚341837361985536 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4911486831748 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0â°iPT2Š- _wandbruntime‚4655  _runtime‚4655ŠßÍ‹Ñ 0loss/FBt1/assignment_loss‚0.5914604291117278 /loss/FRt2/detection_loss‚2.2177424403124077 1loss/SRqqt4/detection_loss‚1.9263112531942235 % _timestamp‚1.7884495395585947e+09 0loss/SRqqt1/detection_loss‚2.608376070857048 1loss/SR¾E4hWqqt2/detection_loss‚1.7269552709425198 %loss/total_loss‚45.52199320150512 /loss/FRt3/assignment_loss‚1.915103892179636 /loss/FRt1/detection_loss‚3.2547592589488397 1loss/SRqqt3/detection_loss‚1.1993898025742082 /loss/FRt1/assignment_loss‚4.999043785608732 0loss/FRt2/assignment_loss‚3.0395081764043765 2loss/SRqqt3/assignment_loss‚0.8615617879426501 2loss/SRqqt4/assignment_loss‚1.2497937800199317 1loss/FBt4/assignment_loss‚0.21888376425367065 _step‚483 .loss/FRt3/detection_loss‚1.579688202073941 .loss/FBt2/detection_loss‚2.533080046827143 .loss/FBt4/detection_loss‚3.121047099431356 epoch‚4 _runtime‚ 4655.03009157 /loss/FRt4/assignment_loss‚2.862820583946851 1loss/FBt2/assignment_loss‚0.19712719744504428 .loss/FRt4/detection_loss‚2.305483792813457 .loss/FBt1/detection_loss‚2.746168488805944 /loss/FBt3/detection_loss‚0.9314561029335102 2loss/SRqqt1/assignment_loss‚1.9640633966773748 2loss/SRqqt2/assignment_loss‚1.3314326732766395 0loss/FBt3/assignment_loss‚0.1407359049238889 trainer/global_step‚5899‘#'N\ŒÖ %loss/total_loss‚45.52199320150512 /loss/FRt3/assignment_loss‚1.915103892179636 /loss/FRt1/detection_loss‚3.2547592589488397 1loss/SRqqt3/detection_loss‚1.1993898025742082 /loss/FRt1/assignment_loss‚4.999043785608732 0loss/FRt2/assignment_loss‚3.0395081764043765 2loss/SRqqt3/assignment_loss‚0.8615617879426501 2loss/SRqqt4/assignment_loss‚1.2497937800199317 1loss/FBt4/assignment_loss‚0.21888376425367065 _step‚483 .loss/FRt3/detection_loss‚1.579688202073941 .loss/FBt2/detection_loss‚2.533080046827143 .loss/FBt4/detection_loss‚3.121047099431356 epoch‚4 _runtime‚ 4655.03009157 /loss/FRt4/assignment_loss‚2.862820583946851 1loss/FBt2/assignment_loss‚0.19712719744504428 .loss/FRt4/detection_loss‚2.305483792813457 .loss/FBt1/detection_loss‚2.746168488805944 /loss/FBt3/detection_loss‚0.9314561029335102 2loss/SRqqt1/assignment_loss‚1.9640633966773748 2loss/SRqqt2/assignment_loss‚1.3314326732766395 0loss/FBt3/assignment_loss‚0.1407359049238889 trainer/global_step‚5899 0loss/FBt1/assignment_loss‚0.5914604291117278 /loss/FRt2/detection_loss‚2.2177424403124077 1loss/SRqqt4/detection_loss‚1.9263112531942235 % _timestamp‚1.7884495395585947e+09 0loss/SRqqt1/detection_loss‚2.608376070857048 1loss/SRqqt2/detection_loss‚1.7269552709425198ãû©ž‚(:“ ¦æÔåÛñü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.985' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.797000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0׎;5oŽ‚(:ä ¦æÔùÎŒÿ disk.rbd1.in‚0% disk.nvme0n1.out‚18459.6806640625 disk.drbd1716.out‚0 disk.drbd2265.in‚0) disk./.usagePercent‚20.22324425322479 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.rbd1.out‚ 24.369140625# memory_percent‚94.2051649093628) proc.memory.percent‚16.13903045654297 cpu‚25.311421062431407 disk.rbd0.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 180.37890625 disk.drbd2265.out‚0 disk.drbd2266.out‚0" disk.dm-0.out‚18459.6806640625 disk.sda.out‚ 180.37890625 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 proc.memory.rssMB‚ 2644.21875 proc.cpu.threads‚17 disk.dm-1.out‚ 180.37890625$ disk./.usageGB‚301.3409538269043 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 network.sent‚ 1.3762238e+07 network.recv‚ 1.700891e+06 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda1.out‚0 disk.dm-0.in‚0.078125( proc.memory.availableMB‚ 949.42578125' disk.nvme0c0n1.out‚18459.68066406254‰žx‚(:’ œ¦æÔù„ý gpu.0.gpu‚46.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚138.28' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.656000000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0- Šr‚(:ç œ¦æÔΑÿ disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875$ memory_percent‚94.20359134674072 cpu‚25.310586924885495 network.recv‚ 1.708094e+06% disk.nvme0n1.out‚18500.8955078125 disk.dm-1.out‚ 181.06640625 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 proc.memory.rssMB‚ 2644.21875* disk./.usagePercent‚20.223836654659024" disk.dm-0.out‚18500.8955078125 disk.sda2.out‚0 disk.rbd1.out‚ 24.634765625 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 181.06640625 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.drbd2080.in‚0( proc.memory.availableMB‚ 949.68359375% disk./.usageGB‚301.34978103637695 disk.drbd2266.out‚0 disk.sda.out‚ 181.06640625 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 network.sent‚ 1.3871336e+07 disk.sda1.out‚0 disk.sda2.in‚0' disk.nvme0c0n1.out‚18500.8955078125) proc.memory.percent‚16.13903045654297öÉÏç2‘- _wandbruntime‚4690  _runtime‚4690ʼ/Ç’Á & lr-AdamW/pg1‚7.606679518252265e-05 & lr-AdamW/pg2‚7.606679518252265e-05 trainer/global_step‚5949 % _timestamp‚1.7884495752493997e+09 _runtime‚4690.721083369 _step‚484­Š ×Ì“Æ & lr-AdamW/pg1‚7.606679518252265e-05 & lr-AdamW/pg2‚7.606679518252265e-05 trainer/global_step‚5949 % _timestamp‚1.7884495752493997e+09 _runtime‚4690.721083369 _step‚484äjPd82”- _wandbruntime‚4690  _runtime‚4690×l )á'•ÛO M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 112042454 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚349568457108992 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 198188402 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚5949 _step‚485 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚344598660889088 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚381602184768000 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚335589304650752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4969139857408 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚381602003495424 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚86125908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚335589287611392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚340540810386736 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚349568457108992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚449 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚344598660889088 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚340540138909620 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚78573737 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚349567782743552 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚376387229255168 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28328978 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚86125910 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7884495752503703e+09 _runtime‚4690.722279055 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚50244310 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚349567782743552 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚86125908 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4969796219904 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 112042454&Yžæ'–àO cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5214955512832 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚86125908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚335589287611392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚340540810386736 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚349568457108992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚449 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚344598660889088 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚340540138909620 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚78573737 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚349567782743552 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚376387229255168 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28328978 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚86125910 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7884495752503703e+09 _runtime‚4690.722279055 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚50244310 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚349567782743552 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚86125908 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4969796219904 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 112042454 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 112042454 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚349568457108992 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 198188402 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚5949 _step‚485 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚344598660889088 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚381602184768000 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚335589304650752 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4969139857408 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚381602003495424 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚349625629588992 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _runtime‚4691.412187551 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚86139924 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 112080972 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚349624955210240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚78585990 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚376448856052736 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4952322171156 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28333514 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5215641176576 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 198200832 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚344654995924992 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚376448879992832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.5¦ large_pool.peak‚ 163111424 _step‚486 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚349625629588992 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚340596530952468 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚344654995924992 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚50252476 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚381664695265792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚344655013927936 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚86139926 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 198220898 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 trainer/global_step‚5949 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 198220898 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚344655013927936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4970615661056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚78586453 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚340595859475144 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 % _timestamp‚1.7884495759398544e+091µ¾\ð&™êM RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚78585990 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚376448856052736 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4952322171156 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚28333514 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5215641176576 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 198200832 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚344654995924992 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚376448879992832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 _step‚486 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚349625629588992 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚340596530952468 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚344654995924992 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚50252476 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚381664695265792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚344655013927936 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚86139926 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 198220898 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 trainer/global_step‚5949 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 198220898 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚344655013927936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4970615661056 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚78586453 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚340595859475144 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 % _timestamp‚1.7884495759398544e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4970615661056 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚463 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 198200832 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 112080972 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚335644208781312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 112060908 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚335644191741952 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚349625629588992 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _runtime‚4691.412187551 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚86139924 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 112080972 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚349624955210240 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚437 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚50652710 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28557352 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚86826708 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚384727518447104 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚347416294828544 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5257776248832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚343326852254144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚86826710 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚379469766138368 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚338334494139392 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚86826708 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 112988544 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚79210060 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚489 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4991686637636 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5010125663232 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28557787 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚338334511178752 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4992341075392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 112988544 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚86826710 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 112968506 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚352426420491776 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 112968506 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚79210497 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5010782025728 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚347416312831488 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5257935678464 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚5999 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚352427094857216 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5010125663232 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚347416294828544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 $ _timestamp‚1.788449612416713e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚347416312831488&îF­º¨ßO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚352427094857216 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5010125663232 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚347416294828544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 $ _timestamp‚1.788449612416713e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.laH1¨“+%rge_pool.allocated‚347416312831488 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚437 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚50652710 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚28557352 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚86826708 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚384727518447104 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚347416294828544 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5257776248832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚343326852254144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚86826710 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚379469766138368 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚338334494139392 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚86826708 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 112988544 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚79210060 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚489 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4991686637636 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5010125663232 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚28557787 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚338334511178752 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4992341075392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 112988544 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚86826710 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 112968506 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚352426420491776 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 112968506 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚79210497 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5010782025728 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚347416312831488 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5257935678464 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚5999éÄk 2©- _wandbruntime‚4728  _runtime‚4728¨Ýÿ«ë&ªåM \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚384790223484416 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚352484267337216 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚352484267337216 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚379531416876032 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚86840724 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚338389398269952 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 199847750 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚86840726 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 112986960 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5011601466880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚426 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5011601466880 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚79222647 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 146833408 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚86840726 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚50660876 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚352483592958464 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 199827684 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 199847750 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7884496131068015e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚347472665870336 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 170773504 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4993157510564 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 199827684 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚343381901342552 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 113007024 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 112986960 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚428 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚343382572819876 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4992503072600 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5258806608384 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 113007024 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 112986960 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚428 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚343382572819876 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4992503072600 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5258806608384 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚338389398269952 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 199847750 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚86840726 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 112986960 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5011601466880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚426 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5011601466880 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚79222647 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 146833408 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚86840726 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚50660876 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚352483592958464 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 199827684 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 199847750 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7884496131068015e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚347472665870336 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 170773504 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4993157510564 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 199827684 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚343381901342552 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944ê›]ÀÓ2¬- _wandbruntime‚4728  _runtime‚4728š—Õ—]­× /loss/FBt2/detection_loss‚1.8694392825875963 trainer/global_step‚5999 _runtime‚4728.579780685 0loss/FBt1/assignment_loss‚0.4433465834881417 1loss/FBt4/assignment_loss‚0.44720418006011187 0loss/SRqqt2/detection_loss‚2.149916258123186 1loss/SRqqt4/detection_loss‚1.6128732617302188 2loss/SRqqt3/assignment_loss‚1.3204796496676663 &loss/total_loss‚47.025771509686145 epoch‚4 /loss/FRt1/assignment_loss‚4.399997635682424 2loss/SRqqt4/assignment_loss‚1.6636692179559633 /loss/FRt2/detection_loss‚2.2354309499263763 .loss/FBt1/detection_loss‚3.265275752544403 % _timestamp‚1.7884496131082826e+09 _step‚491 2loss/SRqqt1/assignment_loss‚2.0271117642521856 .loss/FRt3/detection_loss‚1.768088877321251 /loss/FBt3/detection_loss‚0.6689734805490158 0loss/FRt2/assignment_loss‚3.7333728777037725 1loss/FBt3/assignment_loss‚0.20941692772043832 1loss/SRqqt1/detection_loss‚2.4513788749774297 /loss/FBt4/detection_loss‚3.0106379240751266 1loss/SRqqt3/detection_loss‚1.0483808951222733 0loss/FRt3/assignment_loss‚1.9621440566355182 /loss/FRt1/detection_loss‚3.0474078158537545 /loss/FRt4/detection_loss‚2.3180674361281617 0loss/FRt4/assignment_loss‚3.3923470350198968 1loss/SRqqt2/assignment_loss‚1.540986532304022 0loss/FBt2/assignment_loss‚0.4398242402572115á=hKb®Ü .loss/FRt3/detection_loss‚1.768088877321251 /loss/FBt3/detection_loss‚0.6689734805490158 0loss/FRt2/assignment_loss‚3.7333728777037725 1loss/FBt3/assignment_loss‚0.20941692772043832 1loss/SRqqt1/detection_loss‚2.4513788749774297 /loss/FBt4/detection_loss‚3.0106379240751266 1loss/SRqqt3/detection_loss‚1.0483808951222733 0loss/FRt3/assignment_loss‚1.9621440566355182 /loss/FRt1/detection_loss‚3.0474078158537545 /loss/FRt4/detection_loss‚2.3180674361281617 0loss/FRt4/assignment_loss‚3.3923470350198968 1loss/SRqqt2/assignment_loss‚1.540986532304022 0loss/FBt2/assignment_loss‚0.4398242402572115 /loss/FBt2/detection_loss‚1.8694392825875963 trainer/global_step‚5999 _runtime‚4728.579780685 0loss/FBt1/assignment_loss‚0.4433465834881417 1loss/FBt4/assignment_loss‚0.44720418006011187 0loss/SRqqt2/detection_loss‚2.149916258123186 1loss/SRqqt4/detection_loss‚1.6128732617302188 2loss/SRqqt3/assignment_loss‚1.3204796496676663 &loss/total_loss‚47.025771509686145 epoch‚4 /loss/FRt1/assignment_loss‚4.399997635682424 2loss/SRqqt4/assignment_loss‚1.6636692179559633 /loss/FRt2/detection_loss‚2.2354309499263763 .loss/FBt1/detection_loss‚3.265275752544403 % _timestamp‚1.7884496131082826e+09 _step‚491 2loss/SRqqt1/assignment_loss‚2.0271117642521856ëX¬óמ¯‚(:“ ئæÔÏÎøü gpu.0.gpu‚44.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚141.945' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.388999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ë5´/q°‚(:æ ئæÔÙÔòÿ disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0) proc.memory.percent‚16.13903045654297 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 proc.memory.rssMB‚ 2644.21875 cpu‚25.30719643872397 disk.drbd1716.out‚0 disk.drbd2265.in‚0" disk.dm-0.out‚18670.7802734375 disk.sda2.in‚0 disk.rbd1.out‚ 25.244140625 disk.dm-1.in‚ 0.01171875* disk./.usagePercent‚20.222252736044684% disk./.usageGB‚301.32617950439453' disk.nvme0c0n1.out‚18670.7802734375 disk.sda3.out‚ 183.27734375 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 183.27734375 disk.drbd1716.in‚0$ memory_percent‚94.20421123504639 network.sent‚ 1.3996577e+07 disk.drbd2266.in‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚18670.7802734375( proc.memory.availableMB‚ 949.58203125 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.dm-1.out‚ 183.27734375 network.recv‚ 1.726079e+06 disk.drbd2265.out‚0¼‘“α‚(:… ç¦æÔ¹¸¤ý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.95' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.79 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0šiUk²‚(:à ç¦æÔ„Çìþ disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 network.sent‚ 1.4104305e+07 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.nvme0n1.in‚0.078125 network.recv‚ 1.731626e+06* disk./.usagePercent‚20.223719659215877 disk.sda3.in‚ 0.01171875 disk.dm-0.out‚18726.09765625 disk.sda.in‚ 0.01171875 disk.sda.out‚ 183.63671875 disk.drbd1716.out‚0 disk.drbd2266.out‚0# disk.nvme0n1.out‚18726.09765625 disk.dm-1.out‚ 183.63671875 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.drbd1716.in‚0 cpu‚25.306453712811944 disk.drbd2080.in‚0% disk.nvme0c0n1.out‚18726.09765625 disk.sda1.in‚0& proc.memory.availableMB‚ 949.796875) proc.memory.percent‚16.13903045654297 disk.rbd1.out‚ 25.4013671875 disk.sda3.out‚ 183.63671875 disk.sda2.out‚0 proc.memory.rssMB‚ 2644.21875% disk./.usageGB‚301.34803771972656 disk.rbd0.out‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0$ memory_percent‚94.20289993286133ŽV!峂(:’ ö¦æÔ¹“¬ý gpu.0.gpu‚4.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚186.837' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.367399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0yÞ–5g´‚(:Ü ö¦æԣدÿ$ memory_percent‚94.18280124664307 disk.rbd1.out‚ 25.4951171875* proc.memory.percent‚16.129088401794434 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.sda3.out‚ 184.234375 disk.drbd2266.in‚0 disk.dm-0.out‚ 18763.5078125 network.sent‚ 1.4111264e+07 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.dm-1.out‚ 184.234375$ disk.nvme0c0n1.out‚ 18763.5078125 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125# proc.memory.rssMB‚ 2642.58984375 cpu‚25.313824809330892$ disk./.usageGB‚301.3584403991699 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0( proc.memory.availableMB‚ 953.08984375 network.recv‚ 1.737104e+06* disk./.usagePercent‚20.224417791761752 disk.sda.in‚ 0.01171875 disk.sda.out‚ 184.234375" disk.nvme0n1.out‚ 18763.5078125 disk.drbd2265.in‚0 disk.drbd2265.out‚0P€T󓵂(:ˆ …§æÔƒ¿Žý gpu.0.gpu‚53.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚190.619' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.1238 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0E™CÒh¶‚(:Ý …§æÔý„€ disk.sda1.in‚0 disk.sda1.out‚0 proc.cpu.threads‚17 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.sda2.out‚0$ memory_percent‚94.18494701385498 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875 cpu‚25.3281061824965% disk./.usageGB‚301.37963104248047( proc.memory.availableMB‚ 952.73828125# proc.memory.rssMB‚ 2642.58984375* proc.memory.percent‚16.129088401794434* disk./.usagePercent‚20.225839913614458 disk.rbd1.out‚ 25.5576171875 disk.drbd2080.in‚0& disk.nvme0c0n1.out‚18804.533203125 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 network.sent‚ 1.411689e+07 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 184.84375 disk.drbd2265.out‚0 disk.sda3.out‚ 184.84375 disk.rbd0.in‚0 disk.dm-1.in‚ 0.01171875 network.recv‚ 1.740915e+06 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 184.84375$ disk.nvme0n1.out‚18804.533203125 disk.drbd1716.out‚0 disk.drbd2266.in‚0! disk.dm-0.out‚18804.533203125zSÊÿ“·‚(:ˆ ”§æÔµÂÿü gpu.0.gpu‚54.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚195.716' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚39.1432 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¥¼GŒk¸‚(:à ”§æÔìÝðÿ disk.sda.out‚ 185.515625 network.sent‚ 1.4123253e+07$ disk.nvme0n1.out‚18841.462890625 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125$ disk./.usageGB‚301.3966293334961 disk.rbd1.in‚0 disk.sda2.in‚0) proc.memory.availableMB‚ 1018.01171875 proc.cpu.threads‚17* disk./.usagePercent‚20.226980683187033 disk.rbd1.out‚ 25.5576171875 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 185.515625 disk.drbd2080.in‚0 disk.sda3.out‚ 185.515625 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.rbd0.in‚0$ memory_percent‚93.78654956817627! proc.memory.rssMB‚ 2579.515625 cpu‚25.341791861305275 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 network.recv‚ 1.745506e+06 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125& disk.nvme0c0n1.out‚18841.462890625) proc.memory.percent‚15.74411392211914 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 disk.rbd0.out‚0! disk.dm-0.out‚18841.462890625§È!€2¹- _wandbruntime‚4810  _runtime‚4810Éá[’H*ºÂT FDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 5047248826364 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 202197845 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955586048 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 202181866 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15979 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 5047739740756 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚159 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚347637235140180 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510833152 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 5066391897600 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚342589495399424 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492830208 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚356879615927296 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚351813206026752 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15979 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 5315574679552 % _timestamp‚1.7884496951290386e+09 VHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚ 114196905 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 5066391897600 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 114180928 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚347636727186428 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚88000938 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 _step‚492 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚351813224029696 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 5065899067392 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚51347134 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚351813206026752 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚389477185692160 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚356879105094144 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚389477042213888 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 337538560 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚384161587072512 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 _runtime‚4810.601131092 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚342589478360064 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 QIDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.allocated‚414¥oçM*»ÇT FDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 5047248826364 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 202197845 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955586048 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 202181866 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15979 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 5047739740756 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚159 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚347637235140180 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510833152 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 5066391897600 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚342589495399424 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492830208 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚356879615927296 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚351813206026752 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15979 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 5315574679552 % _timestamp‚1.7884496951290386e+09 VHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚ 114196905 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 5066391897600 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 114180928 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚347636727186428 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 UHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚88000938 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 _step‚492 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚351813224029696 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 5065899067392 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚51347134 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚351813206026752 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚389477185692160 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚356879105094144 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚389477042213888 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 337538560 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚384161587072512 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 _runtime‚4810.601131092 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚342589478360064 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 QIDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.allocated‚414ìëZêM2¼- _wandbruntime‚4811  _runtime‚4811á—sˆ<½ËR OBDeviceStatsMonitor.on_validation_batch_end/active.large_pool.freed‚88011744 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 aMDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.allocated‚389515426552320 _KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.freed‚342624772119552 VGDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak‚ 9148669952 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 MGDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 5066264194048 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚356916811082752 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚159 ]IDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.freed‚389515283074048 _runtime‚4811.543868336 WJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚88011746 XDDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.freed‚347672384555324 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 BDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 QIDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.peak‚889 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚ 5047612435772 WIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 hTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚384199486746112 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚351850036055552 KEDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.freed‚0 $ _timestamp‚1.788449696071902e+09 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚ 114204207 R1lÚýLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 RCDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.peak‚ 9955586048 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 PCDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed‚80349799 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚51353636 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24401 UADeviceStatsMonitor.on_validation_batch_end/active_bytes.all.freed‚356916300249600 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 OEDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 202215953 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 202215953 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 ICDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 M?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚ 202199974 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 bPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚ 5315820268032 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 337538560 TGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚80349960 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 OIDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak‚0 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 LFDeviceStatsMonitor.on_validation_batch_end/oversize_segments.allocated‚0 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914392 WGDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.allocated‚ 10194255872 _step‚493 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 119538176 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 TFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚ 114204207 cODeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.allocated‚342624789158912 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚351850054058496 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚51353638 PBDeviceStatsMonitor.on_validation_batch_end/active.small_pool.freed‚ 114188230 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚351850054058496 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚347672892509076 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 XDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚356916300249600 TFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚ 114188230 BDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 QIDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.peak‚889 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚ 5047612435772 WIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 hTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚384199486746112 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚351850036055552 KEDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.freed‚0 $ _timestamp‚1.788449696071902e+09 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚ 114204207 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 RCDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.peak‚ 9955586048 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 PCDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed‚80349799 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚51353636 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24401 UADeviceStatsMonitor.on_validation_batch_end/active_bytes.all.freed‚356916300249600 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 OEDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 202215953 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 202215953 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 ICDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.freed‚0 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 M?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚ 202199974 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 bPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚ 5315820268032 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 337538560 TGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚80349960 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 OIDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak‚0 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 LFDeviceStatsMonitor.on_validation_batch_end/oversize_segments.allocated‚0 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914392 WGDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.allocated‚ 10194255872 _step‚493 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 119538176 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 TFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚ 114204207 cODeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.allocated‚342624789158912 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚351850054058496 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚51353638 PBDeviceStatsMonitor.on_validation_batch_end/active.small_pool.freed‚ 114188230 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚351850054058496 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚347672892509076 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 XDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚356916300249600 TFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚ 114188230 B&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2291666567325592 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.10968468338251114 particle/accuracy_1_of_2‚1 particle/accuracy_4_of_4‚1 particle/accuracy_7_of_7‚0.5 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5671296715736389 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.30000001192092896 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.2857142984867096 -"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.8125 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.13534483313560486 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5710992217063904 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5483437180519104 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.15732674300670624 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 8 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.47058823704719543 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.2928542494773865 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9851504564285278 *"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.5 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.13168147206306458 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 &particle/f_score‚0.401580810546875 9"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.4696078300476074 ,"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.625 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9766082167625427 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4626705050468445 <"Purity/2FRt*SRqqt2FBt/event_purity‚0.0058139534667134285 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.49960628151893616 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5087910294532776 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 7 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.3055555522441864 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.1041666716337204 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.07559523731470108 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.6041666865348816 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3541666567325592 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 & Purity/4FRt*SRqqt*FBt/FRt_purity‚0 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.966110110282898 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.5222213268280029 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9956028461456299 ,jet/accuracy_4_of_5‚0.004096262622624636 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9635735154151917 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9200000166893005 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9972295761108398 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0020674937404692173 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.04356742650270462 7 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.0939394012093544 particle/accuracy_2_of_3‚1 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.4166666567325592 $ _timestamp‚1.788449709122012e+09 ( Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.36679697036743164 8"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.542801558971405 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.07456140965223312 jet/accuracy_5_of_7‚0 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9764367341995239 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.31741073727607727 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6264368295669556 particle/accuracy_2_of_2‚1 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 ("Purity/1FRt*SRqqt3FBt/event_purity‚0 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.006235827691853046 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9749823808670044 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 & Purity/4FRt1SRqqt0FBt/FRt_purity‚0 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9872402548789978 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.09039430320262909 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.25440531969070435 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.025963719934225082 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.13834549486637115 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7121211886405945 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.15732674300670624 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.1666666716337204 particle/accuracy_1_of_7‚1 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9100529551506042 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.2295287847518921 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 *"Purity/0FRt2SRqqt3FBt/SRqqt_purity‚0.5 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6791017651557922 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5523746609687805 :"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.46404463052749634 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9639881253242493 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.007925670593976974 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9975308775901794 particle/accuracy_1_of_5‚1 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.8684211373329163 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.45773443579673767 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5669847726821899 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 particle/accuracy_2_of_5‚1 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.029201310127973557 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.31937023997306824 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4548611640930176 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.8947991728782654 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 ,"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.625 jet/accuracy_2_of_7‚0.5 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3755350708961487 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9705194234848022 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.10888157784938812 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.4177631437778473 _runtime‚4824.595757467 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5815842151641846 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/0FRt2SRqqt1FBt/FBt_pur×£÷%^ity‚0.9937106966972351 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9602431654930115 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.5833333730697632 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9834415316581726 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.014440140686929226 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.1428571492433548 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.11137402802705765 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9166666865348816 ( Purity/1FRt0SRqqt3FBt/FRt_purity‚0.5 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 particle/accuracy_3_of_3‚1 9"Purity/0FRt0SRqqt3FBt/event_purity‚0.0885416716337204 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0694444477558136 7 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.1587570458650589 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5483437180519104 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 *jet/accuracy_2_of_5‚0.4658986032009125 (particle/accuracy‚0.8520534634590149 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_6_of_7‚1 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1049487367272377 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9812865853309631 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.20264950394630432 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.40019381046295166 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.1027725413441658 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9967948794364929 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 +particle/sensitivity‚0.3181765079498291 particle/accuracy_1_of_6‚1 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9285714030265808 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.3440476357936859 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.07205381989479065 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5191709995269775 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5550214052200317 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8799465298652649 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5490966439247131 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9739475846290588 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9783273935317993 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5534784197807312 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 6 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.183083638548851 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6017342805862427 7 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.2537037432193756 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.13424168527126312 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9842948913574219 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.16430726647377014 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.10440613329410553 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9847255349159241 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6423460841178894 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 jet/accuracy_3_of_7‚0 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 9"Purity/0FRt2SRqqt1FBt/event_purity‚0.0638364776968956 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5496326088905334 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5277777910232544 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9270833730697632 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.3636363744735718 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 jet/accuracy_5_of_5‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 8"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.581464409828186 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.17262569069862366 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.12752054631710052 9"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.4771241545677185 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9855442643165588 6 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.992457389831543 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.3125 ,"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.625 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9974009990692139 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5407681465148926 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 jet/accuracy_6_of_7‚0 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.16893050074577332 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 jet/accuracy_7_of_7‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 *"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5 7 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.8999999761581421 trainer/global_step‚6044 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4766666889190674 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9270833134651184 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.020991317927837372 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 jet/accuracy_1_of_6‚1 particle/accuracy_3_of_7‚1 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.5833333730697632 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.10000000149011612 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.1488485485315323 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 +jet/accuracy_3_of_4‚0.05351520702242851 *jet/accuracy_1_of_5‚0.9578917026519775 particle/accuracy_4_of_6‚1 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9940677881240845 6 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.962654709815979 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5929701924324036 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 particle/accuracy_6_of_6‚1 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.36049386858940125 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9840600490570068 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.09311498701572418 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.1760762482881546 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.4166666567325592 9"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.8999999761581421 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5144708156585693 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 7 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.1162024587392807 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.30000001192092896 jet/accuracy_4_of_7‚0 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.44400128722190857 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 particle/accuracy_5_of_5‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.12602433562278748 8"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.565690815448761 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17216384410858154 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 particle/accuracy_1_of_4‚1 +jet/accuracy_4_of_6‚0.03846153989434242 *"Purity/*FRt2SRqqt3FBt/SRqqt_purity‚0.5 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9822918772697449 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3758775591850281 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.013316193595528603 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 -"Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.3125 particle/accuracy_3_of_6‚1 8"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.576694905757904 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2053571492433548 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5389730334281921 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.06028369069099426 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.34583333134651184 :"Purity/2FRt0SRqqt2FBt/event_purity‚0.00657894741743803 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9778660535812378 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5809981822967529 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.12410497665405273 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9780701994895935 +jet/accuracy_3_of_5‚0.10048004239797592 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.09646400064229965 jet/accuracy_6_of_6‚0 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 epoch‚4 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.029678700491786003 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 9"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.4515015482902527 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.15354938805103302 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.6010293960571289 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.5989118218421936 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 particle/accuracy_1_of_3‚1 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4325776696205139 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.1279817670583725 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9890211820602417 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.31481480598449707 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9911196231842041 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.4717172086238861 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 particle/accuracy_2_of_4‚1 *particle/specificity‚0.950832724571228 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0674603208899498 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.11155320703983307 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 )jet/accuracy_1_of_4‚0.916534423828125 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0694444477558136 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.2419053316116333 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5086420178413391 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ("Purity/1FRt0SRqqt3FBt/event_purity‚0 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3250018060207367 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24684220552444458 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_3‚0.24126437306404114 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 +jet/accuracy_2_of_6‚0.43589746952056885 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.09567900747060776 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 +jet/accuracy_3_of_6‚0.11538461595773697 jet/accuracy_5_of_6‚0 particle/accuracy_5_of_7‚1 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5000446438789368 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.0863625630736351 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5568801164627075 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 particle/accuracy_3_of_4‚1 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5925947427749634 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.23333333432674408 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 7"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.54666668176651 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_7‚1 8"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.514224112033844 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13237082958221436 *jet/accuracy_2_of_4‚0.3654254078865051 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.11363636702299118 particle/accuracy_2_of_7‚1 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.6358025074005127 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.09087394922971725 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.19272783398628235 8"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.527286946773529 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9972295761108398 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5183912515640259 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.17143714427947998 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7424241900444031 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 :"Purity/1FRt2SRqqt0FBt/event_purity‚0.00266400258988142 ,jet/accuracy_4_of_4‚0.002917833859100938 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.5756613612174988 ,jet/accuracy_3_of_3‚0.024638507515192032 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5964912176132202 5 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.22667296230793 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9722471237182617 :"Purity/0FRt1SRqqt2FBt/event_purity‚0.05706075578927994 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.19956403970718384 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.37623274326324463 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5796405076980591 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5556247234344482 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 7validation_average_jet_accuracy‚0.44566285610198975 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.1666666716337204 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.28738167881965637 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 8 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.30000001192092896 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 particle/accuracy_5_of_6‚1 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.2348005473613739 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4674329161643982 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3541666567325592 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.19561728835105896 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9427355527877808 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5603871941566467 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5559027791023254 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4798329770565033 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 particle/accuracy_1_of_1‚1 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3240487575531006 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5329054594039917 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.1299230307340622 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4166666269302368 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5795197486877441 _step‚494 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.039617396891117096 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9913690686225891 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.1341322809457779 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_2_of_6‚1 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5900040864944458 & Purity/*FRt1SRqqt4FBt/FRt_purity‚1Éø†NÍ!ÃÛ¦ 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5598313212394714 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.13333332538604736 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.26827970147132874 6 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.245138481259346 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5845725536346436 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.2803681492805481 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8232211470603943 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.02202867902815342 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5833333730697632 *jet/accuracy_1_of_1‚0.5022261738777161 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5449365377426147 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.2909809648990631 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.10835427045822144 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.8958333730697632 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5580080151557922 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0014041514368727803 *jet/accuracy_1_of_2‚0.7695991396903992 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 *"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 9"Purity/1FRt0SRqqt*FBt/event_purity‚0.1776292473077774 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.11591395735740662 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 & Purity/4FRt1SRqqt*FBt/FRt_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.4443415403366089 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5684009194374084 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 (loss/val_total_loss‚49.1198420629067 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2291666567325592 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.10968468338251114 particle/accuracy_1_of_2‚1 particle/accuracy_4_of_4‚1 particle/accuracy_7_of_7‚0.5 & Purity/2FRt*SRqqt3FBt/FRt_purity‚0 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5671296715736389 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.30000001192092896 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.2857142984867096 -"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.8125 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.13534483313560486 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5710992217063904 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5483437180519104 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.15732674300670624 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 8 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.47058823704719543 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.2928542494773865 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9851504564285278 *"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.5 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.13168147206306458 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 &particle/f_score‚0.401580810546875 9"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.4696078300476074 ,"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.625 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9766082167625427 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4626705050468445 <"Purity/2FRt*SRqqt2FBt/event_purity‚0.0058139534667134285 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.49960628151893616 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5087910294532776 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 7 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.3055555522441864 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.1041666716337204 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.07559523731470108 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.6041666865348816 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3541666567325592 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 & Purity/4FRt*SRqqt*FBt/FRt_purity‚0 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.966110110282898 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.5222213268280029 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9956028461456299 ,jet/accuracy_4_of_5‚0.004096262622624636 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9635735154151917 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9200000166893005 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9972295761108398 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0020674937404692173 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.04356742650270462 7 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.0939394012093544 particle/accuracy_2_of_3‚1 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.4166666567325592 $ _timestamp‚1.788449709122012e+09 ( Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.36679697036743164 8"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.542801558971405 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.07456140965223312 jet/accuracy_5_of_7‚0 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9764367341995239 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.31741073727607727 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6264368295669556 particle/accuracy_2_of_2‚1 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 ("Purity/1FRt*SRqqt3FBt/event_purity‚0 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.006235827691853046 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9749823808670044 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 & Purity/4FRt1SRqqt0FBt/FRt_purity‚0 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9872402548789978 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.09039430320262909 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.25440531969070435 ;"ÇcJ•qPurity/*FRt2SRqqt1FBt/event_purity‚0.025963719934225082 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.13834549486637115 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7121211886405945 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.15732674300670624 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.1666666716337204 particle/accuracy_1_of_7‚1 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9100529551506042 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.2295287847518921 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 *"Purity/0FRt2SRqqt3FBt/SRqqt_purity‚0.5 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6791017651557922 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5523746609687805 :"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.46404463052749634 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9639881253242493 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.007925670593976974 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9975308775901794 particle/accuracy_1_of_5‚1 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.8684211373329163 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.45773443579673767 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5669847726821899 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 particle/accuracy_2_of_5‚1 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.029201310127973557 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.31937023997306824 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4548611640930176 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.8947991728782654 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 ,"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.625 jet/accuracy_2_of_7‚0.5 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3755350708961487 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9705194234848022 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.10888157784938812 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.4177631437778473 _runtime‚4824.595757467 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5815842151641846 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.9937106966972351 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9602431654930115 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.5833333730697632 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9834415316581726 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.014440140686929226 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.1428571492433548 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.11137402802705765 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9166666865348816 ( Purity/1FRt0SRqqt3FBt/FRt_purity‚0.5 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 particle/accuracy_3_of_3‚1 9"Purity/0FRt0SRqqt3FBt/event_purity‚0.0885416716337204 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0694444477558136 7 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.1587570458650589 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5483437180519104 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 *jet/accuracy_2_of_5‚0.4658986032009125 (particle/accuracy‚0.8520534634590149 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_6_of_7‚1 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1049487367272377 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9812865853309631 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.20264950394630432 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.40019381046295166 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.1027725413441658 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9967948794364929 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 +particle/sensitivity‚0.3181765079498291 particle/accuracy_1_of_6‚1 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9285714030265808 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.3440476357936859 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.07205381989479065 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5191709995269775 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5550214052200317 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8799465298652649 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5490966439247131 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9739475846290588 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9783273935317993 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5534784197807312 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 6 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.183083638548851 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6017342805862427 7 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.2537037432193756 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.13424168527126312 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9842948913574219 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.16430726647377014 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.10440613329410553 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9847255349159241 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6423460841178894 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 jet/accuracy_3_of_7‚0 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 9"Purity/0FRt2SRqqt1FBt/event_purity‚0.0638364776968956 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5496326088905334 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5277777910232544 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9270833730697632 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.3636363744735718 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 jet/accuracy_5_of_5‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 8"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.581464409828186 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.17262569069862366 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.12752054631710052 9"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.4771241545677185 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9855442643165588 6 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.992457389831543 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.3125 ,"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.625 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9974009990692139 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5407681465148926 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 jet/accuracy_6_of_7‚0 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.16893050074577332 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 jet/accuracy_7_of_7‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 *"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5 7 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.8999999761581421 trainer/global_step‚6044 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4766666889190674 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9270833134651184 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.020991317927837372 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 jet/accuracy_1_of_6‚1 particle/accuracy_3_of_7‚1 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.5833333730697632 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.10000000149011612 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.1488485485315323 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 +jet/accuracy_3_of_4‚0.05351520702242851 *jet/accuracy_1_of_5‚0.9578917026519775 particle/accuracy_4_of_6‚1 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9940677881240845 6 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.962654709815979 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5929701924324036 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 particle/accuracy_6_of_6‚1 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.36049386858940125 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9840600490570068 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.09311498701572418 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.1760762482881546 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.4166666567325592 9"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.8999999761581421 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 & Purity/2FRt0SRqqt3FBt/FRt_purity‚0 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5144708156585693 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 7 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.1162024587392807 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.30000001192092896 jet/accuracy_4_of_7‚0 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.44400128722190857 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 particle/accuracy_5_of_5‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.12602433562278748 8"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.565690815448761 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17216384410858154 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 particle/accuracy_1_of_4‚1 +jet/accuracy_4_of_6‚0.03846153989434242 *"Purity/*FRt2SRqqt3FBt/SRqqt_purity‚0.5 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9822918772697449 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3758775591850281 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.013316193595528603 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 -"Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.3125 particle/accuracy_3_of_6‚1 8"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.576694905757904 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2053571492433548 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5389730334281921 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.06028369069099426 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 8 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.34583333134651184 :"Purity/2FRt0SRqqt2FBt/event_purity‚0.00657894741743803 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9778660535812378 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5809981822967529 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.12410497665405273 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9780701994895935 +jet/accuracy_3_of_5‚0.10048004239797592 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.09646400064229965 jet/accuracy_6_of_6‚0 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 epoch‚4 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.029678700491786003 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 9"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.4515015482902527 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.15354938805103302 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.6010293960571289 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.5989118218421936 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 particle/accuracy_1_of_3‚1 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4325776696205139 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.1279817670583725 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9890211820602417 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.31481480598449707 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9911196231842041 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.4717172086238861 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 particle/accuracy_2_of_4‚1 *particle/specificity‚0.950832724571228 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0674603208899498 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.11155320703983307 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 )jet/accuracy_1_of_4‚0.916534423828125 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0694444477558136 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.2419053316116333 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5086420178413391 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ("Purity/1FRt0SRqqt3FBt/event_purity‚0 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3250018060207367 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24684220552444458 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_3‚0.24126437306404114 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 +jet/accuracy_2_of_6‚0.43589746952056885 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.09567900747060776 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 +jet/accuracy_3_of_6‚0.11538461595773697 jet/accuracy_5_of_6‚0 particle/accuracy_5_of_7‚1 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5000446438789368 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.0863625630736351 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5568801164627075 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 particle/accuracy_3_of_4‚1 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5925947427749634 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.23333333432674408 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 7"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.54666668176651 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_7‚1 8"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.514224112033844 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13237082958221436 *jet/accuracy_2_of_4‚0.3654254078865051 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.11363636702299118 particle/accuracy_2_of_7‚1 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.6358025074005127 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.09087394922971725 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.19272783398628235 8"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.527286946773529 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9972295761108398 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5183912515640259 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.17143714427947998 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7424241900444031 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 :"Purity/1FRt2SRqqt0FBt/event_purity‚0.00266400258988142 ,jet/accuracy_4_of_4‚0.002917833859100938 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.5756613612174988 ,jet/accuracy_3_of_3‚0.024638507515192032 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5964912176132202 5 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.22667296230793 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9722471237182617 :"Purity/0FRt1SRqqt2FBt/event_purity‚0.05706075578927994 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.19956403970718384 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.37623274326324463 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5796405076980591 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5556247234344482 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 7validation_average_jet_accuracy‚0.44566285610198975 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.1666666716337204 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.28738167881965637 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 8 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.30000001192092896 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 particle/accuracy_5_of_6‚1 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.2348005473613739 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4674329161643982 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3541666567325592 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.19561728835105896 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9427355527877808 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5603871941566467 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5559027791023254 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4798329770565033 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 particle/accuracy_1_of_1‚1 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3240487575531006 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5329054594039917 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.1299230307340622 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4166666269302368 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5795197486877441 _step‚494 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.039617396891117096 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9913690686225891 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.1341322809457779 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_2_of_6‚1 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5900040864944458 & Purity/*FRt1SRqqt4FBt/FRt_purity‚1 8"Purity/*FRt*SRqqt1FBt/event_purity‚0.402335524559021 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.1041666716337204 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9767441749572754 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 jet/accuracy_1_of_7‚1 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 & Purity/4FRt*SRqqt0FBt/FRt_purity‚0 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 particle/accuracy_4_of_5‚1 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.37037041783332825î¶ Ä‚(:’ ²§æÔ°å³€ gpu.0.gpu‚30.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚179.69' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.937999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¾=pÅ‚(:å ²§æÔ涪‚ disk.drbd1716.out‚0 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 186.7265625" proc.memory.rssMB‚ 2621.6484375) disk./.usagePercent‚20.22768342386852 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda.out‚ 186.7265625 disk.nvme0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0* proc.memory.percent‚16.001272201538086 cpu‚25.363115353234697 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0( proc.memory.availableMB‚ 1434.8046875 disk.sda3.out‚ 186.7265625 disk.rbd0.in‚0" disk.dm-0.out‚19410.6298828125 disk.sda2.out‚0$ memory_percent‚91.24264717102051 network.recv‚ 1.756434e+06 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚19410.6298828125 disk.rbd0.out‚0 disk.dm-0.in‚0.078125% disk.nvme0n1.out‚19410.6298828125 proc.cpu.threads‚17 network.sent‚ 1.4237545e+07 disk.rbd1.in‚0 disk.rbd1.out‚ 26.06640625% disk./.usageGB‚301.40710067749023œã‚2Æ- _wandbruntime‚4831  _runtime‚4831|aéóÇÇÁ & lr-AdamW/pg1‚7.427016618219481e-05 & lr-AdamW/pg2‚7.427016618219481e-05 trainer/global_step‚6049 % _timestamp‚1.7884497164125628e+09 _runtime‚4831.884174051 _step‚495iºVÌÈÆ _step‚495 & lr-AdamW/pg1‚7.427016618219481e-05 & lr-AdamW/pg2‚7.427016618219481e-05 trainer/global_step‚6049 % _timestamp‚1.7884497164125628e+09 _runtime‚4831.884174051ïy¹n92É- _wandbruntime‚4831  _runtime‚4831åŽ6/ÊÛO JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5074131439104 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5324280113664 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5056100914882 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚6049 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚348359325469378 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5055446477126 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/ºÃ¨²!reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚357628370024448 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚357628370024448 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚352554256588288 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚348358653992262 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 114376559 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 202564807 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚352554238585344 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚390262754345984 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 114376559 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚357629044389888 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚352554238585344 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5074787801600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚51470860 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚80544450 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚413 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5324101809664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚343303207515136 % _timestamp‚1.7884497164132605e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚88208286 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚357629044389888 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚390262552101888 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚496 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚88208286 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚384938474232320 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚343303224554496ãý6æ'ËàO WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5074131439104 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5324280113664 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5056100914882 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚6049 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚348359325469378 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5055446477126 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚357628370024448 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚357628370024448 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚352554256588288 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚348358653992262 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 114376559 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 202564807 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚352554238585344 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚390262754345984 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 114376559 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚357629044389888 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚352554238585344 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5074787801600 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚51470860 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚80544450 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚413 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5324101809664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚343303207515136 % _timestamp‚1.7884497164132605e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚88208286 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚357629044389888 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚390262552101888 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚496 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚88208286 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚384938474232320 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚343303224554496 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0ð•DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5074950866944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5324946381824 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 202597277 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚357686216869888 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚385000124969984 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚88222302 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚348415046035110 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 114395039 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚357685542491136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚357686216869888 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 202597277 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚497 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚352610591624192 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5325126769664 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚88222304 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚80557114 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 204327936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5075607242752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 202617343 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 114395039 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚6049 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5056917350054 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 114374975 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚390325047411712 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29078518 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 114374975 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 180387840 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚352610609627136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚348414374557786 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚88222304 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚80557548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚352610591624192 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 % _timestamp‚1.7884497171007893e+09 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚390325251739648 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5074950866944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batc½¸Ï‹Æh_end/allocated_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5324946381824 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 202597277 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚357686216869888 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚385000124969984 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚88222302 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚348415046035110 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 114395039 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚357685542491136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚357686216869888 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 202597277 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚497 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚352610591624192 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5325126769664 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚88222304 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚80557114 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 204327936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5075607242752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 202617343 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 114395039 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚6049 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5056917350054 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 114374975 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚390325047411712 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29078518 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 114374975 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 180387840 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚352610609627136ñáKüš2Ï- _wandbruntime‚4832  _runtime‚4832 b4î[ÐÕ 0loss/SRqqt1/detection_loss‚2.630222494403521 1loss/SRqqt2/detection_loss‚2.1220338513021884 0loss/FBt4/assignment_loss‚0.8772702713807424 _runtime‚4832.573868975 /loss/FRt1/assignment_loss‚4.484883854022393 2loss/SRqqt3/assignment_loss‚1.3866887057227069 .loss/FRt3/detection_loss‚1.914853995379347 .loss/FBt1/detection_loss‚3.049259367314252 /loss/FBt2/detection_loss‚2.5134552969382358 2loss/SRqqt1/assignment_loss‚2.0237648896872997 1loss/FBt1/assignment_loss‚0.34957268290591775 /loss/FRt1/detection_loss‚3.1135787001022925 1loss/SRqqt4/detection_loss‚1.6955567804120837 /loss/FBt3/detection_loss‚0.8196887188953244 .loss/FBt4/detection_loss‚3.342482626438141 %loss/total_loss‚48.43930470801865 epoch‚5 1loss/FBt2/assignment_loss‚0.41902523774361267 _step‚498 0loss/FRt3/assignment_loss‚1.6534880720652068 0loss/FRt4/assignment_loss‚3.3936907575841535 2loss/SRqqt2/assignment_loss‚1.4068309970847939 1loss/SRqqt3/detection_loss‚1.0443873481042143 trainer/global_step‚6049 0loss/FRt2/assignment_loss‚3.6043313628918416 1loss/SRqqt4/assignment_loss‚1.497320424528822 1loss/FBt3/assignment_loss‚0.30957333124306236 /loss/FRt2/detection_loss‚2.6970847410124703 .loss/FRt4/detection_loss‚2.090260200856025 % _timestamp‚1.7884497171023767e+092„V`ÑÚ 0loss/SRqqt1/detection_loss‚2.630222494403521 1loss/SRqqt2/detection_loss‚2.1220338513021884 0loss/FBt4/assignment_loss‚0.8772702713807424 _runtime‚4832.573868975 /loss/FRt1/assignment_loss‚4.484883854022393 2loss/SRqqt3/assignment_loss‚1.3866887057227069 .loss/FRt3/detection_loss‚1.914853995379347 .loss/FBt1/detection_loss‚3.049259367314252 /loss/FBt2/detection_loss‚2.5134552969382358 2loss/SRqqt1/assignment_loss‚2.0237648896872997 1loss/FBt1/assignment_loss‚0.34957268290591775 /loss/FRt1/detection_loss‚3.1135787001022925 1loss/SRqqt4/detection_loss‚1.6955567804120837 /loss/FBt3/detection_loss‚0.8196887188953244 .loss/FBt4/detection_loss‚3.342482626438141 %loss/total_loss‚48.43930470801865 epoch‚5 1loss/FBt2/assignment_loss‚0.41902523774361267 _step‚498 0loss/FRt3/assignment_loss‚1.6534880720652068 0loss/FRt4/assignment_loss‚3.3936907575841535 2loss/SRqqt2/assignment_loss‚1.4068309970847939 1loss/SRqqt3/detection_loss‚1.0443873481042143 trainer/global_step‚6049 0loss/FRt2/assignment_loss‚3.6043313628918416 1loss/SRqqt4/assignment_loss‚1.497320424528822 1loss/FBt3/assignment_loss‚0.30957333124306236 /loss/FRt2/detection_loss‚2.6970847410124703 .loss/FRt4/detection_loss‚2.090260200856025 % _timestamp‚1.7884497171023767e+09ò„ßCžÒ‚(:“ Á§æÔÿ­ý gpu.0.gpu‚44.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚155.691' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚31.138199999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ÿ‡OtÓ‚(:é Á§æÔ£ó€ disk.sda2.out‚0% disk.nvme0n1.out‚19460.2919921875' disk.nvme0c0n1.out‚19460.2919921875 disk.drbd2266.in‚0* proc.memory.percent‚16.001272201538086* disk./.usagePercent‚20.228971909788353 disk.drbd1716.out‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda1.out‚0 cpu‚25.362124791730125 disk.drbd2080.in‚0 disk.sda.out‚ 187.50390625 disk.rbd1.in‚0$ memory_percent‚91.24305248260498 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 187.50390625 disk.sda3.out‚ 187.50390625 disk.rbd1.out‚ 26.12890625 network.sent‚ 1.4360756e+07 network.recv‚ 1.761968e+06 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125" proc.memory.rssMB‚ 2621.6484375 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2265.out‚0) proc.memory.availableMB‚ 1434.73828125 proc.cpu.threads‚17$ disk./.usageGB‚301.4263000488281 disk.drbd2080.out‚0 disk.rbd0.out‚0" disk.dm-0.out‚19460.2919921875 disk.sda2.in‚0 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.sda1.in‚0OמԂ(:“ ЧæÔàà”ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚138.651' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.730200000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0þyØÓsÕ‚(:è ЧæÔ»‚ÿ$ memory_percent‚91.24462604522705 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125* proc.memory.percent‚16.001272201538086) disk./.usagePercent‚20.22630559131924 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0" proc.memory.rssMB‚ 2621.6484375 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚19966.4404296875 network.recv‚ 1.768878e+06% disk./.usageGB‚301.38656997680664 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125" disk.dm-0.out‚19966.4638671875 disk.rbd1.out‚ 26.21484375 disk.drbd2265.out‚0 disk.drbd2266.out‚0 proc.cpu.threads‚17 disk.sda1.in‚0% disk.nvme0n1.out‚19966.4404296875 disk.drbd1716.out‚0 disk.sda3.in‚ 0.01171875) proc.memory.availableMB‚ 1434.48046875 disk.drbd2080.out‚0 disk.sda.out‚ 188.00390625 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.sda3.out‚ 188.00390625 cpu‚25.361176699451836 network.sent‚ 1.446969e+07 disk.rbd0.in‚0 disk.dm-1.out‚ 188.00390625 disk.drbd2266.in‚0Ñïhk2Ö- _wandbruntime‚4868  _runtime‚4868á»>ûÇ×Á & lr-AdamW/pg2‚7.336834209431963e-05 trainer/global_step‚6099 % _timestamp‚1.7884497532461295e+09 _runtime‚4868.717720807 _step‚499 & lr-AdamW/pg1‚7.336834209431963e-05´Í™ÌØÆ trainer/global_step‚6099 % _timestamp‚1.7884497532461295e+09 _runtime‚4868.717720807 _step‚499 & lr-AdamW/pg1‚7.336834209431963e-05 & lr-AdamW/pg2‚7.336834209431963e-05ó4CåÐ2Ù- _wandbruntime‚4868  _runtime‚4868°®û‚á'ÚÛO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚81177935 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚355371908530688 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚360487007772672 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5115117244928 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 % _timestamp‚1.7884497532468457e+09 _runtime‚4868.718621116 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 115282573 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚355371890527744 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚51879262 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚393387808376832 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5096936254290 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚500 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚88909086 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚346048431082496 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 115302611 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚388020987175424 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚393388004329472 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5366993213952 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116Äò<ð&ÛàO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚360487682138112 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚88909088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚388021011115520 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5366821201408 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚355371890527744 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5096936254290 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚500 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚88909086 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚346048431082496 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 115302611 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚388020987175424 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚393388004329472 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5366993213952 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚81177935 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚355371908530688 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚360487007772672 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5115117244928 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 % _timestamp‚1.7884497532468457e+09 _runtime‚4868.718621116 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 115282573 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚355371890527744 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚51879262 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚393387808376832 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5367667586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚346103318173696 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 204224129 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚346103335213056 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5367839585280 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5115936672768 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚88923102 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚51887430 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚388082637913088 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚393450305499136 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5097098251498 % _timestamp‚1.7884497539355364e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 115321091 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚393450501438464 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 trainer/global_step‚6099 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚360544854618112 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚388082661853184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29303160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚4869.407791482 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚351200416425194 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5115936672768 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29303611 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚351201087902518 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚88923104 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚81190588 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚81191041 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 204244195 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5097752689462 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚51887428 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚355428243566592 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚355428261569536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚81190588 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚81191041 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 204244195 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5097752689462 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚51887428 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚355428243566592 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚355428261569536 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5367667586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚346103318173696 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 204224129 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚346103335213056 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5367839585280 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5115936672768 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚88923102 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚51887430 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚388082637913088 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚393450305499136 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5097098251498 % _timestamp‚1.7884497539355364e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 115321091 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚393450501438464 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 trainer/global_step‚6099 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚360544854618112 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚388082661853184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29303160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚4869.407791482 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚351200416425194 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5115936672768 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29303611 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚351201087902518 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872õ ¦·W2ß- _wandbruntime‚4869  _runtime‚48692Áy8ZàÔ 2loss/SRqqt1/assignment_loss‚2.1442890787124633 0loss/SRqqt3/detection_loss‚1.147589974415799 /loss/FBt4/detection_loss‚3.0327123403549194 2loss/SRqqt2/assignment_loss‚1.7426196929553281 0loss/FBt3/assignment_loss‚0.2755122980012436 /loss/FRt2/detection_loss‚2.0953562928409113 trainer/global_step‚6099 $ _timestamp‚1.788449753937268e+09 0loss/FRt1/assignment_loss‚5.0759961128234865 2loss/SRqqt3/assignment_loss‚1.2762590234892235 0loss/FBt2/assignment_loss‚0.4937986408670601 epoch‚5 0loss/FRt4/assignment_loss‚3.1951506522794566 1loss/SRqqt1/detection_loss‚2.8869295144081115 1loss/SRqqt2/detection_loss‚2.0174563974142075 _step‚502 /loss/FRt3/assignment_loss‚1.362097064299243 2loss/SRqqt4/assignment_loss‚1.6803727554516834 0loss/FBt4/assignment_loss‚0.2325658369809389 1loss/SRqqt4/detection_loss‚1.5975571608072834 .loss/FBt1/detection_loss‚2.726719915866852 %loss/total_loss‚47.41806632326301 _runtime‚ 4869.40906681 0loss/FRt2/assignment_loss‚3.7107936754459288 /loss/FRt4/detection_loss‚2.1550674391910434 /loss/FBt2/detection_loss‚2.1019964242974916 /loss/FBt3/detection_loss‚0.8946810879243108 1loss/FBt1/assignment_loss‚0.41840856762137263 .loss/FRt1/detection_loss‚3.453932875394821 /loss/FRt3/detection_loss‚1.7002035014198296bù¬_áÙ 0loss/FRt1/assignment_loss‚5.0759961128234865 2loss/SRqqt3/assignment_loss‚1.2762590234892235 0loss/FBt2/assignment_loss‚0.4937986408670601 epoch‚5 0loss/FRt4/assignment_loss‚3.1951506522794566 1loss/SRqqt1/detection_loss‚2.8869295144081115 1loss/SRqqt2/detection_loss‚2.0174563974142075 _step‚502 /loss/FRt3/assignment_loss‚1.362097064299243 2loss/SRqqt4/assignment_loss‚1.6803727554516834 0loss/FBt4/assignment_loss‚0.2325658369809389 1loss/SRqqt4/detection_loss‚1.5975571608072834 .loss/FBt1/detection_loss‚2.726719915866852 %loss/total_loss‚47.41806632326301 _runtime‚ 4869.40906681 0loss/FRt2/assignment_loss‚3.7107936754459288 /loss/FRt4/detection_loss‚2.1550674391910434 /loss/FBt2/detection_loss‚2.1019964242974916 /loss/FBt3/detection_loss‚0.8946810879243108 1loss/FBt1/assignment_loss‚0.41840856762137263 .loss/FRt1/detection_loss‚3.453932875394821 /loss/FRt3/detection_loss‚1.7002035014198296 2loss/SRqqt1/assignment_loss‚2.1442890787124633 0loss/SRqqt3/detection_loss‚1.147589974415799 /loss/FBt4/detection_loss‚3.0327123403549194 2loss/SRqqt2/assignment_loss‚1.7426196929553281 0loss/FBt3/assignment_loss‚0.2755122980012436 /loss/FRt2/detection_loss‚2.0953562928409113 trainer/global_step‚6099 $ _timestamp‚1.788449753937268e+09öt£C’â‚(:‡ ߧæÔ⌬þ gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.615' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.523 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¼ˆN«Xã‚(:Í ߧæÔà­Çÿ disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.sda.out‚ 188.60546875 disk.rbd1.in‚0 disk.nvme0c0n1.out‚20010.5 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125( proc.memory.availableMB‚ 1433.8984375" proc.memory.rssMB‚ 2621.6484375* disk./.usagePercent‚20.227394391362363 disk.rbd0.in‚0$ memory_percent‚91.24817848205566 proc.cpu.threads‚17 network.recv‚ 1.772156e+06 disk.nvme0n1.out‚20010.5% disk./.usageGB‚301.40279388427734 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 26.40234375 disk.sda3.out‚ 188.60546875 disk.dm-0.in‚0.078125 disk.dm-0.out‚20010.5 disk.dm-1.out‚ 188.60546875 disk.drbd1716.out‚0 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0* proc.memory.percent‚16.001272201538086 cpu‚25.36022131570085 network.sent‚ 1.4474891e+07 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0¬H›Çžä‚(:“ î§æÔ˜ßûü gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.765' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.952999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¦>£`å‚(:Õ î§æÔÿ disk.dm-1.out‚ 189.28125 disk.drbd2080.in‚0$ memory_percent‚91.25022888183594 disk.drbd2265.out‚0 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 189.28125 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0* proc.memory.percent‚16.001272201538086 cpu‚25.35927888220061 proc.cpu.threads‚17 network.recv‚ 1.780035e+06 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0% disk.nvme0c0n1.out‚20051.80859375 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda1.out‚0 disk.sda2.out‚0* disk./.usagePercent‚20.229170827642456 disk.rbd1.out‚ 26.40234375 disk.sda.out‚ 189.28125" proc.memory.rssMB‚ 2621.6484375$ disk./.usageGB‚301.4292640686035 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-0.out‚20051.80859375 network.sent‚ 1.4584129e+07 disk.sda.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.drbd2265.in‚0# disk.nvme0n1.out‚20051.80859375% proc.memory.availableMB‚ 1433.5625f–v6“æ‚(:ˆ ý§æÔ¯€’ý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.223' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6446 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÚiÚ^ç‚(:Ó ý§æÔ¥——ÿ$ disk./.usageGB‚301.4479446411133 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2621.6484375 disk.nvme0n1.in‚0.078125$ memory_percent‚91.25192165374756 proc.cpu.threads‚17) disk./.usagePercent‚20.23042449653763! disk.nvme0c0n1.out‚ 20100.6875 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 cpu‚25.358339260956043 disk.drbd2265.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.drbd1716.out‚0* proc.memory.percent‚16.001272201538086 disk.drbd2080.in‚0 disk.sda3.out‚ 189.8046875 disk.sda.out‚ 189.8046875 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.dm-0.out‚ 20100.6875 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.out‚0 network.recv‚ 1.783328e+06 disk.sda3.in‚ 0.01171875 disk.nvme0n1.out‚ 20100.6875 network.sent‚ 1.4589313e+07 disk.drbd2266.out‚0 disk.rbd1.out‚ 26.46484375 disk.dm-1.out‚ 189.8046875) proc.memory.availableMB‚ 1433.28515625FØaS2è- _wandbruntime‚4905  _runtime‚4905|à·Åé¿ $ _timestamp‚1.788449789989126e+09 _runtime‚ 4905.46072869 _step‚503 & lr-AdamW/pg1‚7.246442870816366e-05 & lr-AdamW/pg2‚7.246442870816366e-05 trainer/global_step‚6149Ǩ*:ÊêÄ & lr-AdamW/pg2‚7.246442870816366e-05 trainer/global_step‚6149 $ _timestamp‚1.788449789989126e+09 _runtime‚ 4905.46072869 _step‚503 & lr-AdamW/pg1‚7.246442870816366e-05÷2¯ø2ë- _wandbruntime‚4905  _runtime‚4905VÏ´ÈìÛO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29524419 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚89609886 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚89609888 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5137117155942 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚363346319886336 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 116208625 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚504 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚52287660 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5409655682560 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 116228663 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚363345645520896 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚348793637610496 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5156103050752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚89609888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚89609886 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 116228663 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 205818511 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚396513179741184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521‘åh9æ'íàO UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚4905.461578985 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚89609888 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5137117155942 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚363346319886336 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 116208625 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚504 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚52287660 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5409655682560 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 116228663 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚363345645520896 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚348793637610496 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5156103050752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚89609888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚89609886 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 116228663 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 205818511 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚396513179741184 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29524419 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚89609886 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚358245895508992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚396575692081152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚363403492366336 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚358245895508992 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚353987129769926 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚89623904 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚348848524701696 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5157578854400 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5156922478592 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 205850981 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚81824735 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5137933590906 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚4906.141292528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚358245913511936 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚348848541741056 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚89623902 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚81824296 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚391165198736384 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚363402817987584 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 116247143 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 % _timestamp‚1.7884497906691287e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚358245913511936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5138588028870 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5157578854400 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚52295828 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚353986458292602 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚6149 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29528905 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 205871047 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 205850981 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30Ü"6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚358245895508992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚396575692081152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚363403492366336 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚358245895508992 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚353987129769926 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚89623904 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚348848524701696 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5157578854400 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5156922478592 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 205850981 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚81824735 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5137933590906 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚4906.141292528 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚358245913511936 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚348848541741056 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚89623902 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚81824296 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚391165198736384 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚363402817987584 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 116247143 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 % _timestamp‚1.7884497906691287e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚358245913511936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5138588028870 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5157578854400 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚52295828 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚353986458292602 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚6149 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29528905 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 205871047 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 205850981 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30ùFZÁÓ2ñ- _wandbruntime‚4906  _runtime‚4906QɽXòÒ .loss/FBt3/detection_loss‚0.652460442589862 epoch‚5 /loss/FRt2/assignment_loss‚3.661753752292731 1loss/SRqqt3/detection_loss‚1.4072160835109524 2loss/SRqqt1/assignment_loss‚1.8127151890657842 2loss/SRqqt2/assignment_loss‚1.6944022192459167 /loss/FRt2/detection_loss‚2.2389874099156795 _runtime‚4906.142383387 .loss/FRt4/detection_loss‚2.145614952864972 0loss/SRqqt2/detection_loss‚1.739038233356229 /loss/FBt2/detection_loss‚1.7467603305975596 0loss/FRt3/assignment_loss‚1.3863628733725775 0loss/FBt1/assignment_loss‚0.6605304648371451 1loss/FBt3/assignment_loss‚0.20569340247563273 trainer/global_step‚6149 /loss/FRt1/detection_loss‚3.0154657199464996 /loss/FRt3/detection_loss‚1.8308249493794782 0loss/SRqqt1/detection_loss‚2.704624659381807 1loss/SRqqt4/detection_loss‚1.8388620102405548 &loss/total_loss‚45.898129264901364 /loss/FRt1/assignment_loss‚5.086775191899004 .loss/FBt1/detection_loss‚2.758493407206102 /loss/FBt4/detection_loss‚3.1230810642242433 $ _timestamp‚1.788449790670848e+09 _step‚506 0loss/FRt4/assignment_loss‚3.0608086735010147 2loss/SRqqt4/assignment_loss‚1.2129216438159347 0loss/FBt2/assignment_loss‚0.4639252029359317 1loss/SRqqt3/assignment_loss‚1.095138788116717 0loss/FBt4/assignment_loss‚0.3556726001290372M™\å]ó× 0loss/FBt1/assignment_loss‚0.6605304648371451 1loss/FBt3/assignment_loss‚0.20569340247563273 trainer/global_step‚6149 /loss/FRt1/detection_loss‚3.0154657199464996 /loss/FRt3/detection_loss‚1.8308249493794782 0loss/SRqqt1/detection_loss‚2.704624659381807 1loss/SRqqt4/detection_loss‚1.8388620102405548 &loss/total_loss‚45.898129264901364 /loss/FRt1/assignment_loss‚5.086775191899004 .loss/FBt1/detection_loss‚2.758493407206102 /loss/FBt4/detection_loss‚3.1230810642242433 $ _timestamp‚1.788449790670848e+09 _step‚506 0loss/FRt4/assignment_loss‚3.0608086735010147 2loss/SRqqt4/assignment_loss‚1.2129216438159347 0loss/FBt2/assignment_loss‚0.4639252029359317 1loss/SRqqt3/assignment_loss‚1.095138788116717 0loss/FBt4/assignment_loss‚0.3556726001290372 .loss/FBt3/detection_loss‚0.652460442589862 epoch‚5 /loss/FRt2/assignment_loss‚3.661753752292731 1loss/SRqqt3/detection_loss‚1.4072160835109524 2loss/SRqqt1/assignment_loss‚1.8127151890657842 2loss/SRqqt2/assignment_loss‚1.6944022192459167 /loss/FRt2/detection_loss‚2.2389874099156795 _runtime‚4906.142383387 .loss/FRt4/detection_loss‚2.145614952864972 0loss/SRqqt2/detection_loss‚1.739038233356229 /loss/FBt2/detection_loss‚1.7467603305975596 0loss/FRt3/assignment_loss‚1.3863628733725775úõ×ç“ô‚(:ˆ Œ¨æÔÆ·™ý gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚144.623' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚28.9246 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÞÛ¸¢fõ‚(:Û Œ¨æÔØͬ€ disk.drbd2265.in‚0 disk.drbd2080.in‚0 cpu‚25.357392429118992 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.sda2.out‚0$ memory_percent‚91.26286506652832 disk.rbd0.out‚0 disk.sda.out‚ 190.2421875 disk.drbd1716.out‚0 network.recv‚ 1.788392e+06 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 190.2421875 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.drbd2080.out‚0* disk./.usagePercent‚20.231456462908152 disk.dm-0.in‚0.078125 disk.dm-0.out‚ 20139.2734375 disk.rbd1.out‚ 26.46484375 network.sent‚ 1.4595782e+07" disk.nvme0n1.out‚ 20139.2734375* proc.memory.percent‚16.001272201538086 proc.cpu.threads‚17# disk./.usageGB‚301.463321685791 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.dm-1.out‚ 190.2421875 disk.drbd2265.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875$ disk.nvme0c0n1.out‚ 20139.2734375 disk.sda2.in‚0( proc.memory.availableMB‚ 1431.4921875" proc.memory.rssMB‚ 2621.6484375 disk.drbd1716.in‚0‹ªF‘ö‚(:† ›¨æÔÛ×ðü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚181.62' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.324 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0-DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚52696062 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚356717451071602 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5177952495350 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29749221 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚432 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 117154715 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚366204957634560 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚366204283269120 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 207465403 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚399638702015488 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 $ _timestamp‚1.788449826707433e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5197088856576 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 207445363 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚90310686 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚356716779594486 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚6199 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚361007212415488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 117134677 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚82445715 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚52696060 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚399638516548608 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5178606933106 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚366204283269120 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 117134677 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚52696062 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚356717451071602 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5177952495350 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚29749221 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚432 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 117154715 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚366204957634560 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚366204283269120 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 207465403 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚399638702015488 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 $ _timestamp‚1.788449826707433e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5197088856576 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 207445363 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚90310686 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚356716779594486 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚6199 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚361007212415488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 117134677 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚82445715 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚52696060 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚399638516548608 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5178606933106 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚366204283269120 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 117134677 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 117153131 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 207497899 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5198564660224 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29754+_åß_128 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5197908284416 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 117153131 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚52704230 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚361063547451392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 117173195 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚90324702 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _step‚509 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚90324702 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5453480914944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚366261455735808 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚394247735619584 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5178768930314 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 207497899 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚356772500160010 _runtime‚4942.889825855 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚82457930 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚90324704 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5197908284416 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚356773171637334 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚361063565454336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚90324704 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0þ±Gð&€êM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5178768930314 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 207497899 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚356772500160010 _runtime‚4942.889825855 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚82457930 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚90324704 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5197908284416 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚356773171637334 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚361063565454336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚90324704 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 207477833 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5453319401472 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚428 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5179423368278 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚361063565454336 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29753702 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 117153131 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 207497899 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5198564660224 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚29754128 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5197908284416 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 117153131 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚52704230 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚361063547451392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 117173195 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚90324702 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _step‚509 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚90324702 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5453480914944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚366261455735808 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚394247735619584 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5238731024896 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5238074662400 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5238074662400 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚442 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚91011486 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5495659559424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5495498032640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚53104460 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚359502821461894 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚91011488 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚363824846354944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚359503492939010 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚369063595382784 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _runtime‚4979.257514243 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5218787834758 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚91011488 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29974862 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 118080767 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 209072215 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚369062921017344 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚369062921017344 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5238731024896 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5238074662400 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5238074662400 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚442 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚91011486 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5495659559424 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5495498032640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚53104460 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚359502821461894 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚91011488 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚363824846354944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚359503492939010 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚369063595382784 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _runtime‚4979.257514243 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5218787834758 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚91011488 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚29974862 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 118080767 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 209072215 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚83091529 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 209124751 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚83091971 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 118079183 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 209104685 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚442 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 118079183 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚53112630 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚354338954797056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚402826598851584 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚397330272502784 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5239550466048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚363881199393792 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 118099247 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.788449864487557e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29978901 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5238894090240 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 209124751 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚402826792693760 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚91025504 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚397330248562688 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5496350288896 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚363881217396736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5238894090240 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 118099247 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚91025504 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚91025502 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5219604269722 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚53112628 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚359558542027418 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚4979.959533237 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 209104685 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.788449864487557e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚29978901 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5238894090240 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 209124751 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚402826792693760 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚91025504 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚397330248562688 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5496350288896 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚363881217396736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5238894090240 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 118099247 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚91025504 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚91025502 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5219604269722 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚53112628 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚359558542027418 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚4979.959533237 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 209104685 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚83091529 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 209124751 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚83091971 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 118079183 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 209104685 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚442 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 118079183 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚531126309”Á2‘- _wandbruntime‚4979  _runtime‚49791’þ‘?’Ó /loss/FRt2/assignment_loss‚3.693714657555456 0loss/FRt4/assignment_loss‚2.7135174469701173 0loss/FBt1/assignment_loss‚0.3659877516088622 1loss/SRqqt2/detection_loss‚1.8334253424025597 0loss/SRqqt3/detection_loss‚1.267245049572597 .loss/FRt1/detection_loss‚3.377174194369997 epoch‚5 2loss/SRqqt2/assignment_loss‚1.5021119476791391 0loss/FBt4/assignment_loss‚0.8670805275440217 1loss/SRqqt4/detection_loss‚1.6350349138180416 0loss/FRt1/assignment_loss‚4.6606104998361495 .loss/FRt2/detection_loss‚1.844073240523753 /loss/FBt1/detection_loss‚2.5137218420322123 .loss/FBt2/detection_loss‚2.156774401664734 _step‚514 2loss/SRqqt1/assignment_loss‚2.1491042835747494 0loss/FBt2/assignment_loss‚0.6438213298097253 1loss/SRqqt1/detection_loss‚3.5335984966334175 $loss/total_loss‚46.3421291780759 % _timestamp‚1.7884498644891138e+09 _runtime‚ 4979.96061584 0loss/FRt3/assignment_loss‚1.7187883467585952 2loss/SRqqt3/assignment_loss‚0.9943003679025855 1loss/FBt3/assignment_loss‚0.21550079274165815 2loss/SRqqt4/assignment_loss‚1.4112680575810372 /loss/FRt3/detection_loss‚1.7636607083043567 /loss/FRt4/detection_loss‚2.0161462675394684 /loss/FBt3/detection_loss‚0.7139138410471983 /loss/FBt4/detection_loss‚2.7515548706054687  ÉXÒtrainer/global_step‚6249¦¢ü^“Ø /loss/FRt2/assignment_loss‚3.693714657555456 0loss/FRt4/assignment_loss‚2.7135174469701173 0loss/FBt1/assignment_loss‚0.3659877516088622 1loss/SRqqt2/detection_loss‚1.8334253424025597 0loss/SRqqt3/detection_loss‚1.267245049572597 .loss/FRt1/detection_loss‚3.377174194369997 epoch‚5 2loss/SRqqt2/assignment_loss‚1.5021119476791391 0loss/FBt4/assignment_loss‚0.8670805275440217 1loss/SRqqt4/detection_loss‚1.6350349138180416 0loss/FRt1/assignment_loss‚4.6606104998361495 .loss/FRt2/detection_loss‚1.844073240523753 /loss/FBt1/detection_loss‚2.5137218420322123 .loss/FBt2/detection_loss‚2.156774401664734 _step‚514 2loss/SRqqt1/assignment_loss‚2.1491042835747494 0loss/FBt2/assignment_loss‚0.6438213298097253 1loss/SRqqt1/detection_loss‚3.5335984966334175 $loss/total_loss‚46.3421291780759 % _timestamp‚1.7884498644891138e+09 _runtime‚ 4979.96061584 0loss/FRt3/assignment_loss‚1.7187883467585952 2loss/SRqqt3/assignment_loss‚0.9943003679025855 1loss/FBt3/assignment_loss‚0.21550079274165815 2loss/SRqqt4/assignment_loss‚1.4112680575810372 /loss/FRt3/detection_loss‚1.7636607083043567 /loss/FRt4/detection_loss‚2.0161462675394684 /loss/FBt3/detection_loss‚0.7139138410471983 /loss/FBt4/detection_loss‚2.7515548706054687 trainer/global_step‚6249‚iΈXœ”‚(:‘ ȨæÔ‹–þ gpu.0.gpu‚40.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚173.8' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚34.760000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0VˆNq•‚(:æ ȨæÔòð¯ÿ disk.drbd2265.in‚0( disk.nvme0c0n1.out‚20324.42333984375 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875$ memory_percent‚91.26758575439453" proc.memory.rssMB‚ 2621.6484375$ disk./.usageGB‚301.4391326904297 disk.drbd2265.out‚0 disk.sda3.out‚ 192.4296875 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.drbd1716.in‚0 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.dm-1.out‚ 192.4296875 cpu‚25.353670699429905 network.recv‚ 1.8097e+06 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.drbd1716.out‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚20324.42333984375 network.sent‚ 1.4824961e+07 disk.sda.out‚ 192.4296875 disk.sda2.out‚0 disk.rbd1.out‚ 26.9267578125* disk./.usagePercent‚20.229833119133538 disk.sda2.in‚0& proc.memory.availableMB‚ 1430.71875* proc.memory.percent‚16.001272201538086 disk.drbd2266.in‚0 disk.rbd0.out‚0# disk.dm-0.out‚20324.42333984375 disk.sda1.out‚0ÕdK¿“–‚(:ˆ רæÔ ®˜ý gpu.0.gpu‚49.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚171.621' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.3242 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Цaíj—‚(:ß רæÔìàþ disk.sda.out‚ 192.8828125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2265.out‚0( proc.memory.availableMB‚ 1430.8984375 network.recv‚ 1.81527e+06 disk.sda.in‚ 0.01171875 disk.drbd2265.in‚0 cpu‚25.35272087455028 disk.dm-0.in‚0.078125 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2621.6484375 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.sda2.out‚0$ memory_percent‚91.26648902893066* proc.memory.percent‚16.001272201538086 disk.sda2.in‚0 disk.rbd1.out‚ 27.2001953125 disk.dm-1.in‚ 0.01171875% disk./.usageGB‚301.48641204833984 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.dm-1.out‚ 192.8828125) disk./.usagePercent‚20.23300607651291 disk.sda1.out‚0# disk.nvme0n1.out‚20371.10546875 network.sent‚ 1.4932683e+07 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 192.8828125 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.dm-0.out‚20371.10546875 disk.rbd1.in‚0% disk.nvme0c0n1.out‚20371.10546875HsüÞ˜‚(:“ æ¨æÔ³íîü gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.191' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.038199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0赡i™‚(:Þ æ¨æÔ¼çñþ cpu‚25.35176874772942 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0% disk.nvme0c0n1.out‚20414.55078125 disk.dm-0.out‚20414.55078125 disk.sda2.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚301.5111503601074 disk.drbd1716.out‚0 network.sent‚ 1.493943e+07 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.out‚0$ memory_percent‚91.26949310302734# disk.nvme0n1.out‚20414.55078125 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875& proc.memory.availableMB‚ 1430.40625) disk./.usagePercent‚20.23466628537246 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2621.6484375 disk.sda.out‚ 193.66796875 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.sda3.out‚ 193.66796875 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0* proc.memory.percent‚16.001272201538086 network.recv‚ 1.82051e+06 disk.dm-1.out‚ 193.66796875 disk.drbd2080.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 27.2001953125]/Øä2š- _wandbruntime‚5016  _runtime‚5016¢›4ÂÅ›¿ _runtime‚5016.181013593 _step‚515 % lr-AdamW/pg1‚6.97420384433686e-05 % lr-AdamW/pg2‚6.97420384433686e-05 trainer/global_step‚6299 % _timestamp‚1.7884499007093415e+09ðTì ÊœÄ _step‚515 % lr-AdamW/pg1‚6.97420384433686e-05 % lr-AdamW/pg2‚6.97420384433686e-05 trainer/global_step‚6299 % _timestamp‚1.7884499007093415e+09 _runtime‚5016.181013593ƒ¦î²þ2- _wandbruntime‚5016  _runtime‚5016¯ŽŠá'žÛO SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5259623174166 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 118986781 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚53512862 _step‚516 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 210699067 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5279716830720 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚53512860 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30199729 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5538649807872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚6299 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚91712288 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5279716830720 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚91712286 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5538490378240 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚371922233131008 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚91712288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚400351134708224 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚400351158648320 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 _runtime‚5016.181940883 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 210719107 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 119006819 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 118986781 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚371921558765568 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚91712286 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚357029240155136 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 % _timestamp‚1.7884499007100565e+09 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 119006819 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚371921558765568 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚366642516300288 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30199293 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5279060468224 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚357029257194496 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚366642498297344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚366642516300288 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚83712591 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚405889808456192 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728½ŽÖæ'ŸàO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 210699067 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5279716830720 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚53512860 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30199729 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5538649807872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚6299 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚91712288 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5279716830720 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚91712286 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5538490378240 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚371922233131008 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚91712288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚400351134708224 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚400351158648320 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 _runtime‚5016.181940883 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 210719107 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 119006819 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 118986781 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚371921558765568 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚91712286 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚357029240155136 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 % _timestamp‚1.7884499007100565e+09 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 119006819 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚371921558765568 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚366642516300288 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚30199293 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5279060468224 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚357029257194496 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚366642498297344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚366642516300288 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚83712591 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚405889808456192 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5259623174166 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 118986781 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚53512862 _step‚516„âxOò2 - _wandbruntime‚5016  _runtime‚5016¸¶²¡äM _step‚517 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 119025299 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5280536271872 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 119025299 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚366698869339136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚83724750 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5260439609130 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5539513093632 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5279879896064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚400412785445888 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 119005235 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚91726302 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚366698869339136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 210751603 $ _timestamp‚1.788449901410744e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚53521028 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚400412809385984 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚371979405611008 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚405952130734592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 L>DeviceStatsMonitor.on_t¢#> Ïrain_batch_end/active_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚91726302 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30204156 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚366698851336192 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5539345288704 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5280536271872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 trainer/global_step‚6299 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚83725186 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 119005235 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚362345255372150 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚405952322479616 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚91726304 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚91726304õ]Ò³ï&¢éM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚400412785445888 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 119005235 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚91726302 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚366698869339136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 210751603 $ _timestamp‚1.788449901410744e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚53521028 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚400412809385984 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚371979405611008 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚405952130734592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚91726302 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30204156 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚366698851336192 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5539345288704 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5280536271872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 trainer/global_step‚6299 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚83725186 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 119005235 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚362345255372150 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚405952322479616 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚91726304 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚91726304 _step‚517 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 119025299 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5280536271872 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 119025299 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚366698869339136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚83724750 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5260439609130 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5539513093632 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5279879896064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928…–Ö¸2£- _wandbruntime‚5016  _runtime‚5016Ç\šÞX¤Ò % _timestamp‚1.7884499014126763e+09 _runtime‚5016.884192305 1loss/FBt1/assignment_loss‚0.18167613669510713 .loss/FRt2/detection_loss‚2.244512812764037 epoch‚5 trainer/global_step‚6299 2loss/SRqqt3/assignment_loss‚1.0050941462725846 2loss/SRqqt4/assignment_loss‚1.2760975769321834 0loss/FBt2/assignment_loss‚0.4351205060568943 .loss/FRt1/detection_loss‚2.923891191482544 1loss/SRqqt2/detection_loss‚1.7457797999604274 /loss/FRt1/assignment_loss‚4.305643444061279 /loss/FRt3/assignment_loss‚1.718518168368238 1loss/SRqqt1/assignment_loss‚2.289027146556798 1loss/FBt3/assignment_loss‚0.23558688218378973 1loss/SRqqt1/detection_loss‚2.6401341066640964 $loss/total_loss‚45.5302771244818 2loss/SRqqt2/assignment_loss‚1.6994394425113322 1loss/FBt4/assignment_loss‚0.48170466045598914 1loss/SRqqt3/detection_loss‚0.9946993135694248 /loss/FRt2/assignment_loss‚3.888474943591099 /loss/FRt4/assignment_loss‚2.836216819736193 .loss/FRt4/detection_loss‚2.086282381190444 /loss/FRt3/detection_loss‚1.7901307019464514 1loss/SRqqt4/detection_loss‚1.4460102976965052 /loss/FBt1/detection_loss‚3.1629379299970775 /loss/FBt2/detection_loss‚2.3471611936887107 /loss/FBt3/detection_loss‚0.7424037187687449 .loss/FBt4/detection_loss‚3.053733803331852 _step‚518_f§ß]¥× /loss/FRt2/assignment_loss‚3.888474943591099 /loss/FRt4/assignment_loss‚2.836216819736193 .loss/FRt4/detection_loss‚2.086282381190444 /loss/FRt3/detection_loss‚1.7901307019464514 1loss/SRqqt4/detection_loss‚1.4460102976965052 /loss/FBt1/detection_loss‚3.1629379299970775 /loss/FBt2/detection_loss‚2.3471611936887107 /loss/FBt3/detection_loss‚0.7424037187687449 .loss/FBt4/detection_loss‚3.053733803331852 _step‚518 % _timestamp‚1.7884499014126763e+09 _runtime‚5016.884192305 1loss/FBt1/assignment_loss‚0.18167613669510713 .loss/FRt2/detection_loss‚2.244512812764037 epoch‚5 trainer/global_step‚6299 2loss/SRqqt3/assignment_loss‚1.0050941462725846 2loss/SRqqt4/assignment_loss‚1.2760975769321834 0loss/FBt2/assignment_loss‚0.4351205060568943 .loss/FRt1/detection_loss‚2.923891191482544 1loss/SRqqt2/detection_loss‚1.7457797999604274 /loss/FRt1/assignment_loss‚4.305643444061279 /loss/FRt3/assignment_loss‚1.718518168368238 1loss/SRqqt1/assignment_loss‚2.289027146556798 1loss/FBt3/assignment_loss‚0.23558688218378973 1loss/SRqqt1/detection_loss‚2.6401341066640964 $loss/total_loss‚45.5302771244818 2loss/SRqqt2/assignment_loss‚1.6994394425113322 1loss/FBt4/assignment_loss‚0.48170466045598914 1loss/SRqqt3/detection_loss‚0.9946993135694248†›6íF“¦‚(:ˆ õ¨æÔ‰‘ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.503' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.9006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¢8Œ@r§‚(:ç õ¨æÔ¥Áÿ# memory_percent‚91.2667989730835 network.sent‚ 1.4944563e+07 disk.sda1.in‚0* proc.memory.percent‚16.001272201538086 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda2.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 194.30859375 disk.drbd1716.in‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 194.30859375 disk.dm-0.in‚0.078125" proc.memory.rssMB‚ 2621.6484375 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0) proc.memory.availableMB‚ 1430.84765625 disk.sda1.out‚0$ disk.nvme0n1.out‚20457.486328125 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 194.30859375! disk.dm-0.out‚20457.486328125 network.recv‚ 1.823904e+06$ disk./.usageGB‚301.5065574645996 disk.drbd2265.out‚0 cpu‚25.350920105686686 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875* disk./.usagePercent‚20.234358052301232 disk.rbd1.out‚ 27.2626953125 disk.drbd2080.out‚0& disk.nvme0c0n1.out‚20457.486328125 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0´ìH$’¨‚(:‡ „©æÔô—Àý gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚184.337' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8674 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“3A‘s©‚(:è „©æÔ¶Ôòþ disk.rbd1.out‚ 27.2626953125 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚ 0.01171875* proc.memory.percent‚16.001272201538086$ disk.nvme0n1.out‚20496.951171875 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda2.in‚0" proc.memory.rssMB‚ 2621.6484375! disk.dm-0.out‚20496.951171875 network.sent‚ 1.5053567e+07 disk.drbd2265.in‚0 disk.sda3.out‚ 194.89453125 network.recv‚ 1.830715e+06 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 194.89453125 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda2.out‚0 cpu‚25.350032107708714 proc.cpu.threads‚17* disk./.usagePercent‚20.239134896882625 disk.nvme0n1.in‚0.078125& disk.nvme0c0n1.out‚20496.951171875 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125$ memory_percent‚91.26832485198975$ disk./.usageGB‚301.5777359008789 disk.drbd2266.out‚0 disk.sda.out‚ 194.89453125) proc.memory.availableMB‚ 1430.59765625d£|©2ª- _wandbruntime‚5052  _runtime‚5052`ñlÅ«¿ & lr-AdamW/pg1‚6.883165537076362e-05 & lr-AdamW/pg2‚6.883165537076362e-05 trainer/global_step‚6349 $ _timestamp‚1.788449937526631e+09 _runtime‚ 5052.99833836 _step‚519P7"Ê¬Ä trainer/global_step‚6349 $ _timestamp‚1.788449937526631e+09 _runtime‚ 5052.99833836 _step‚519 & lr-AdamW/pg1‚6.883165537076362e-05 & lr-AdamW/pg2‚6.883165537076362e-05‡Ÿb³2­- _wandbruntime‚5052  _runtime‚5052c´—-á'®ÛO VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚92413086 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚409015103118336 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚359774463722496 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚369460150239744 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 119912833 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚92413088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚53921262 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚92413088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚84345956 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚446 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5581239768576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚365074905196710 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚6349 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚403433695531520 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚374780196513792 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚369460168242688 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5320046274048 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 212325919 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5320702636544 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚53921260 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚374780196513792 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 119932871 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5301112951330 _step‚520’<ûæ¯àO ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 119912833 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚359774463722496 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚369460150239744 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 119912833 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚92413088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚53921262 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚92413088 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚84345956 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚446 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5581239768576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚365074905196710 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚6349 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚403433695531520 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚374780196513792 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚369460168242688 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5320046274048 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 212325919 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5320702636544 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚53921260 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚374780196513792 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 119932871 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5301112951330 _step‚520 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚92413086 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚409015103118336 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 212378455 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚409077612860416 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5320865701888 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚403495322329088 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚5053.696966841 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚403495346269184 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5320865701888 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚369516521281536 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30429621 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5301274948538 % _timestamp‚1.7884499382248993e+09 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 119951351 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚365131297239558 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚359829367853056 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 212378455 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚444 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚84358607 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚359829350813696 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 212358389 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 119931287 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚92427104 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚53929428 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚369516503278592 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚92427102 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 119951351 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _step‚521 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚365130625762234ÃëŒ:ð&²êM H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚444 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚84358607 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚359829350813696 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 212358389 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 119931287 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚92427104 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚53929428 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚369516503278592 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚92427102 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 119951351 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _step‚521 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚365130625762234 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 212378455 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚409077612860416 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5320865701888 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚403495322329088 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚5053.696966841 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚403495346269184 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5320865701888 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚369516521281536 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30429621 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5301274948538 % _timestamp‚1.7884499382248993e+09 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 119951351 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚365131297239558 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚359829367853056 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 212378455‰Ä6 2³- _wandbruntime‚5053  _runtime‚5053ºíW´Ñ .loss/FRt1/assignment_loss‚4.40532763004303 1loss/SRqqt2/detection_loss‚2.2035904796542347 1loss/SRqqt2/assignment_loss‚1.608908192229432 1loss/SRqqt4/detection_loss‚1.5895725023001432 _step‚522 % _timestamp‚1.7884499382265851e+09 /loss/FRt2/detection_loss‚2.1094417697853514 0loss/FRt2/assignment_loss‚3.4016069995032416 0loss/FRt3/assignment_loss‚2.4622344348741616 /loss/FRt4/assignment_loss‚2.625039720310355 0loss/FBt1/assignment_loss‚0.5409075514401643 1loss/SRqqt1/detection_loss‚2.5776633378118277 2loss/SRqqt1/assignment_loss‚2.1155766351148486 0loss/FBt2/assignment_loss‚0.3862954540178028 0loss/SRqqt3/detection_loss‚1.163518023746672 /loss/FBt2/detection_loss‚2.0953783631324767 /loss/FBt3/detection_loss‚0.8148177171152242 .loss/FBt4/detection_loss‚3.045683813095093 %loss/total_loss‚46.31495202933532 /loss/FRt1/detection_loss‚3.2604120552539824 1loss/SRqqt4/assignment_loss‚1.297406939224207 1loss/FBt3/assignment_loss‚0.23083432160196093 -loss/FRt3/detection_loss‚1.98428457044065 /loss/FRt4/detection_loss‚2.1824717479494384 .loss/FBt1/detection_loss‚2.585192564460966 2loss/SRqqt3/assignment_loss‚1.1652943165277811 epoch‚5 1loss/FBt4/assignment_loss‚0.46349288970227465 trainer/global_step‚6349 _runtime‚5053.698156175À‡Ë\µÖ .loss/FRt1/assignment_loss‚4.40532763004303 1loss/SRqqt2/detection_loss‚2.2035904796542347 1loss/SRqqt2/assignment_loss‚1.608908192229432 1loss/SRqqt4/detection_loss‚1.5895725023001432 _step‚522 % _timestamp‚1.7884499382265851e+09 /loss/FRt2/detection_loss‚2.1094417697853514 0loss/FRt2/assignment_loss‚3.4016069995032416 0loss/FRt3/assignment_loss‚2.4622344348741616 /loss/FRt4/assignment_loss‚2.625039720310355 0loss/FBt1/assignment_loss‚0.5409075514401643 1loss/SRqqt1/detection_loss‚2.5776633378118277 2loss/SRqqt1/assignment_loss‚2.1155766351148486 0loss/FBt2/assignment_loss‚0.3862954540178028 0loss/SRqqt3/detection_loss‚1.163518023746672 /loss/FBt2/detection_loss‚2.0953783631324767 /loss/FBt3/detection_loss‚0.8148177171152242 .loss/FBt4/detection_loss‚3.045683813095093 %loss/total_loss‚46.31495202933532 /loss/FRt1/detection_loss‚3.2604120552539824 1loss/SRqqt4/assignment_loss‚1.297406939224207 1loss/FBt3/assignment_loss‚0.23083432160196093 -loss/FRt3/detection_loss‚1.98428457044065 /loss/FRt4/detection_loss‚2.1824717479494384 .loss/FBt1/detection_loss‚2.585192564460966 2loss/SRqqt3/assignment_loss‚1.1652943165277811 epoch‚5 1loss/FBt4/assignment_loss‚0.46349288970227465 trainer/global_step‚6349 _runtime‚5053.698156175ŠÝq<ž¶‚(:“ “©æÔõ —þ gpu.0.gpu‚49.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.737' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.547399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚07$j_·‚(:æ “©æÔ☀ disk.drbd2080.in‚0 disk.drbd2080.out‚0" disk.dm-0.out‚20540.7490234375 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 27.419921875 disk.drbd1716.out‚0 cpu‚25.349135282410046 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125& proc.memory.availableMB‚ 1432.03125$ disk./.usageGB‚301.5921287536621 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚20540.7490234375 disk.sda3.out‚ 195.26953125 disk.drbd2266.in‚0$ memory_percent‚91.25957489013672 network.sent‚ 1.5058679e+07* proc.memory.percent‚16.001272201538086 proc.cpu.threads‚17 disk.sda3.in‚ 0.01171875% disk.nvme0n1.out‚20540.7490234375 disk.dm-1.out‚ 195.26953125 network.recv‚ 1.833758e+06 disk.rbd0.out‚0 disk.sda.out‚ 195.26953125 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0) disk./.usagePercent‚20.24010081330931 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0" proc.memory.rs÷(¸ÌsMB‚ 2621.6484375‹€rÏ“¸‚(:ˆ ¢©æÔ½›—ý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.413' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.6826 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Æî¬Jp¹‚(:å ¢©æÔ“¬“€ network.sent‚ 1.5167205e+07 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 195.9140625 disk.dm-1.out‚ 195.9140625" proc.memory.rssMB‚ 2621.6484375 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚301.5985527038574 disk.drbd2080.out‚0" disk.dm-0.out‚20580.4951171875 disk.rbd1.in‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0( proc.memory.availableMB‚ 1431.9921875 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125* proc.memory.percent‚16.001272201538086 disk.rbd0.out‚0 disk.sda.out‚ 195.9140625 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 cpu‚25.34825401771528 network.recv‚ 1.840912e+06* disk./.usagePercent‚20.240531929996973 disk.drbd1716.in‚0 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚20580.4951171875 disk.rbd1.out‚ 27.419921875% disk.nvme0n1.out‚20580.4951171875$ memory_percent‚91.25981330871582PÀz¡“º‚(:ˆ ±©æÔÀ‰…ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.701' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.9402 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€ƒ)°q»‚(:æ ±©æÔÊÉ—ÿ* disk./.usagePercent‚20.241922562922966 disk.dm-0.in‚0.078125# disk.dm-0.out‚20612.59619140625 disk.rbd1.out‚ 27.482421875& disk.nvme0n1.out‚20612.59619140625 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 196.6796875" proc.memory.rssMB‚ 2621.6484375 disk.drbd2265.out‚0 disk.sda.out‚ 196.6796875 disk.rbd1.in‚0( disk.nvme0c0n1.out‚20612.59619140625 disk.sda3.out‚ 196.6796875 network.sent‚ 1.5172356e+07 disk.rbd0.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0( proc.memory.availableMB‚ 957.49609375 network.recv‚ 1.8439e+06 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.drbd2080.out‚0$ memory_percent‚94.15590763092041* proc.memory.percent‚16.001272201538086 cpu‚25.34734025295037$ disk./.usageGB‚301.6192741394043 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 proc.cpu.threads‚17 disk.sda2.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda1.in‚0 disk.drbd2266.out‚0Ä2¼- _wandbruntime‚5090  _runtime‚5090¨ÁÀǽÁ & lr-AdamW/pg1‚6.792012843859125e-05 & lr-AdamW/pg2‚6.792012843859125e-05 trainer/global_step‚6399 % _timestamp‚1.7884499747952511e+09 _runtime‚5090.266811566 _step‚523®ˆÄ Ì¾Æ _runtime‚5090.266811566 _step‚523 & lr-AdamW/pg1‚6.792012843859125e-05 & lr-AdamW/pg2‚6.792012843859125e-05 trainer/global_step‚6399 % _timestamp‚1.7884499747952511e+09‹ä³Q2¿- _wandbruntime‚5090  _runtime‚5090PšPJá'ÀÛO QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 213952771 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5361032079872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚367861618541234 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 120838885 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235328 _step‚524 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚54329660 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚54329662 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚372277802182144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚406516232414720 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5624164002304 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30650397 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚377638834262016 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚93113888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚362519653211136 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179175424 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 _runtime‚5090.267738505 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5341948290738 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚84979616 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚54329662 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚372277802182144 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚406516232414720 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5624164002304 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30650397 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚377638834262016 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚93113888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚362519653211136 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179175424 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 _runtime‚5090.267738505 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5341948290738 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚84979616 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚84992245 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚54337830 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚362574574381056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚372334155220992 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5361851507712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5342764725910 % _timestamp‚1.7884499754887419e+09 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5342110287946 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30654417 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 120857339 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_b9ÝOytes.all.allocated‚412203067332608 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5625016375296 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 120877403 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5362507883520 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30654860 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 213985241 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 214005307 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 213985241 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚412202875587584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚63998‘ûÍð&ÄêM \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5362507883520 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚30654860 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 213985241 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 214005307 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 213985241 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚412202875587584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚6399 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚93127902 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚372334155220992 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 120877403 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _step‚525 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚377696681107456 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚84992245 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚54337830 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚362574574381056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚372334155220992 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5361851507712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5342764725910 % _timestamp‚1.7884499754887419e+09 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5342110287946 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30654417 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 120857339 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚412203067332608 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5625016375296 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 120877403 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808lŸíH2Å- _wandbruntime‚5090  _runtime‚5090ù²£MXÆÒ 1loss/SRqqt2/assignment_loss‚1.470639510323172 0loss/FBt3/assignment_loss‚0.2419764801731361 1loss/SRqqt1/detection_loss‚2.5278043610709053 epoch‚5 trainer/global_step‚6399 0loss/FBt1/assignment_loss‚0.3539498762518936 0loss/FBt4/assignment_loss‚0.3030403210182807 2loss/SRqqt4/assignment_loss‚1.6592252940560381 0loss/FBt2/assignment_loss‚0.4744592170754913 /loss/FRt4/detection_loss‚2.3637538238977776 /loss/FBt4/detection_loss‚3.4126745462417603 /loss/FRt1/assignment_loss‚4.705941819227659 1loss/SRqqt1/assignment_loss‚2.236498970644815 /loss/FRt1/detection_loss‚3.0595679512390723 1loss/SRqqt2/detection_loss‚1.7073149574191675 1loss/SRqqt4/detection_loss‚1.9185121251891057 _runtime‚5090.961653286 0loss/FRt4/assignment_loss‚2.9806745801216517 2loss/SRqqt3/assignment_loss‚1.0455446800459987 /loss/FRt3/detection_loss‚1.7099232573527843 .loss/FRt2/detection_loss‚2.038376253686453 .loss/FRt2/assignment_loss‚3.32193680342875 1loss/SRqqt3/detection_loss‚1.0835878098032612 /loss/FBt3/detection_loss‚0.6987376057787945 $ _timestamp‚1.788449975490183e+09 _step‚526 0loss/FRt3/assignment_loss‚1.0919115422293544 /loss/FBt1/detection_loss‚2.8508055061101913 /loss/FBt2/detection_loss‚2.3919838964939117 %loss/total_loss‚45.64884118887943À3•Â]Ç× 1loss/SRqqt2/assignment_loss‚1.470639510323172 0loss/FBt3/assignment_loss‚0.2419764801731361 1loss/SRqqt1/detection_loss‚2.5278043610709053 epoch‚5 trainer/global_step‚6399 0loss/FBt1/assignment_loss‚0.3539498762518936 0loss/FBt4/assignment_loss‚0.3030403210182807 2loss/SRqqt4/assignment_loss‚1.6592252940560381 0loss/FBt2/assignment_loss‚0.4744592170754913 /loss/FRt4/detection_loss‚2.3637538238977776 /loss/FBt4/detection_loss‚3.4126745462417603 /loss/FRt1/assignment_loss‚4.705941819227659 1loss/SRqqt1/assignment_loss‚2.236498970644815 /loss/FRt1/detection_loss‚3.0595679512390723 1loss/SRqqt2/detection_loss‚1.7073149574191675 1loss/SRqqt4/detection_loss‚1.9185121251891057 _runtime‚5090.961653286 0loss/FRt4/assignment_loss‚2.9806745801216517 2loss/SRqqt3/assignment_loss‚1.0455446800459987 /loss/FRt3/detection_loss‚1.7099232573527843 .loss/FRt2/detection_loss‚2.038376253686453 .loss/FRt2/assignment_loss‚3.32193680342875 1loss/SRqqt3/detection_loss‚1.0835878098032612 /loss/FBt3/detection_loss‚0.6987376057787945 $ _timestamp‚1.788449975490183e+09 _step‚526 0loss/FRt3/assignment_loss‚1.0919115422293544 /loss/FBt1/detection_loss‚2.8508055061101913 /loss/FBt2/detection_loss‚2.3919838964939117 %loss/total_loss‚45.64884118887943Žràà“È‚(:ˆ À©æÔíº™ý gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.194' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.6388 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0â{ÄÅsÉ‚(:è À©æÔÞË¡ÿ disk.sda1.in‚0 disk.dm-1.out‚ 197.38671875 disk.sda3.out‚ 197.38671875 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.rbd1.in‚0" proc.memory.rssMB‚ 2621.6484375 disk.sda3.in‚ 0.01171875 disk.sda2.in‚0% disk.nvme0n1.out‚20649.8642578125 disk.dm-1.in‚ 0.01171875 proc.cpu.threads‚17$ disk./.usageGB‚301.6258659362793 disk.drbd2080.in‚0" disk.dm-0.out‚20649.8642578125 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 cpu‚25.346469870735664 network.recv‚ 1.848711e+06 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 197.38671875 disk.rbd1.out‚ 27.568359375 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.out‚0* disk./.usagePercent‚20.242364943942132' disk.nvme0c0n1.out‚20649.8642578125 disk.drbd1716.in‚0 disk.drbd2265.out‚0$ memory_percent‚94.15764808654785' proc.memory.availableMB‚ 957.2109375 network.sent‚ 1.5178822e+07 disk.drbd1716.out‚0* proc.memory.percent‚16.001272201538086³0³“Ê‚(:ˆ Ï©æÔ˜“ý gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚175.203' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.0406 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0|¥ŠtË‚(:é Ï©æÔ¾Îíþ disk.dm-1.out‚ 197.83984375 disk.sda3.out‚ 197.83984375 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.rbd1.out‚ 27.841796875 disk.nvme0n1.in‚0.078125$ disk./.usageGB‚301.6326370239258 disk.drbd2080.out‚0 disk.drbd2266.out‚0# disk.dm-0.out‚20717.04443359375 disk.sda1.out‚0* proc.memory.percent‚16.001272201538086 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 cpu‚25.345609585678243 disk.dm-1.in‚ 0.01171875# memory_percent‚94.1561222076416 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚20717.04443359375 disk.sda1.in‚0' proc.memory.availableMB‚ 957.4609375" proc.memory.rssMB‚ 2621.6484375 disk.sda2.out‚0 network.sent‚ 1.5286546e+07 disk.sda.out‚ 197.83984375 disk.sda2.in‚0 disk.drbd2265.out‚0 proc.cpu.threads‚17 network.recv‚ 1.854522e+06 disk.rbd0.in‚0 disk.dm-0.in‚0.078125& disk.nvme0n1.out‚20717.04443359375) disk./.usagePercent‚20.24281935731541(¹à 2Ì- _wandbruntime‚5126  _runtime‚5126Ï6äáÆÍÀ _step‚527 & lr-AdamW/pg1‚6.700764760769798e-05 & lr-AdamW/pg2‚6.700764760769798e-05 trainer/global_step‚6449 % _timestamp‚1.7884500111837025e+09 _runtime‚ 5126.65529209W25>ËÎÅ trainer/global_step‚6449 % _timestamp‚1.7884500111837025e+09 _runtime‚ 5126.65529209 _step‚527 & lr-AdamW/pg1‚6.700764760769798e-05 & lr-AdamW/pg2‚6.700764760769798e-05\Îyê2Ï- _wandbruntime‚5126  _runtime‚5126ó&õÆá'ÐÛO UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚446 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚409598745357824 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5667260854272 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5382129192390 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884500111844676e+09 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5402674248192 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5667090938880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚380498146375680 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚415266030152192 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚375095454124544 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 215599663 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚380497472010240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚375095472127488 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5402017885696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 121784975 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚370646988931526 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 121784975 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚365264859739136 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 121764937 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚93814688 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5382783630146 trainer/global_step‚6449 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚93814688 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5402674248192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚380498146375680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 121764937 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30876965 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚85614579 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 _runtime‚5126.656357962 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚54738062 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚375095472127488 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚448 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚93814686 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚409598769297920 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5402017885696 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚375095454124544 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚380498146375680 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚415266030152192 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚375095454124544 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 215599663 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚380497472010240 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚375095472127488 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5402017885696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 121784975 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚370646988931526 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 121784975 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚365264859739136 G?DeviceStatsMonitor.on_traiÿ÷%FXn_batch_start/segment.large_pool.peak‚430 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 121764937 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚93814688 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5382783630146 trainer/global_step‚6449 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚93814688 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5402674248192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚380498146375680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 121764937 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚30876965 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚85614579 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 _runtime‚5126.656357962 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚54738062 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚375095472127488 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚448 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚93814686 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚409598769297920 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5402017885696 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚375095454124544 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚365319763869696 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 % _timestamp‚1.7884500118799222e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5668110798336 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚365319780909056 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5667945090560 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚370703380974374 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 121783391 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚415328530833920 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5382945627354 trainer/global_step‚6449 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 215612093 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚93828702 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚85627676 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 121783391 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚375151825166336 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5402837313536 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚93828702 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚54746228 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚380554644476928 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚380555318855680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 215632159 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 215612093 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚375151825166336 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 121803455 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚370702709497050 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚380555318855680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 121803455 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚85627226 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30880998 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚415328341186048 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚93828704 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚365319763869696 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 % _timestamp‚1.7884500118799222e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5668110798336 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚365319780909056 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5667945090560 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚370703380974374 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 121783391 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚415328530833920 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5382945627354 trainer/global_step‚6449 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 215612093 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚93828702 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚85627676 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 121783391 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚375151825166336 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5402837313536 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚93828702 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚54746228 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚380554644476928 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚380555318855680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 215632159 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 215612093 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚375151825166336 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 121803455 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚370702709497050 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚380555318855680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 121803455 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚85627226 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚30880998 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚415328341186048 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324‘ˆ,€2Õ- _wandbruntime‚5127  _runtime‚5127 M1XÖÒ /loss/FRt1/assignment_loss‚4.812425480948554 1loss/SRqqt2/assignment_loss‚1.381329893074449 0loss/FBt4/assignment_loss‚0.4586617611344839 1loss/SRqqt1/detection_loss‚2.7574049631754556 2loss/SRqqt4/assignment_loss‚1.8726518608274914 0loss/FBt1/assignment_loss‚0.6151587087551889 .loss/FRt2/detection_loss‚2.280585476756096 .loss/FRt3/detection_loss‚1.997559439142545 trainer/global_step‚6449 /loss/FRt1/detection_loss‚3.5362469024128385 0loss/FRt3/assignment_loss‚1.8011647356881035 1loss/SRqqt4/detection_loss‚1.6213807991099736 _step‚530 /loss/FBt3/detection_loss‚0.7588852436488701 /loss/FBt4/detection_loss‚2.7542316913604736 %loss/total_loss‚46.74386327034141 % _timestamp‚1.7884500118816137e+09 /loss/FRt2/assignment_loss‚3.632868893444538 2loss/SRqqt1/assignment_loss‚2.3095653344083718 0loss/FBt2/assignment_loss‚0.6207417413178413 0loss/FBt3/assignment_loss‚0.2172303041166711 0loss/FRt4/assignment_loss‚2.8630882183710735 1loss/SRqqt3/assignment_loss‚1.062575309283354 /loss/FRt4/detection_loss‚2.4437388519446057 /loss/FBt1/detection_loss‚2.1780409465233483 epoch‚5 _runtime‚5127.353101028 1loss/SRqqt2/detection_loss‚1.8497322841036705 1loss/SRqqt3/detection_loss‚1.1429439543774633 .loss/FBt2/detection_loss‚1.775650476415952¿X]×× /loss/FRt1/detection_loss‚3.5362469024128385 0loss/FRt3/assignment_loss‚1.8011647356881035 1loss/SRqqt4/detection_loss‚1.6213807991099736 _step‚530 /loss/FBt3/detection_loss‚0.7588852436488701 /loss/FBt4/detection_loss‚2.7542316913604736 %loss/total_loss‚46.74386327034141 % _timestamp‚1.7884500118816137e+09 /loss/FRt2/assignment_loss‚3.632868893444538 2loss/SRqqt1/assignment_loss‚2.3095653344083718 0loss/FBt2/assignment_loss‚0.6207417413178413 0loss/FBt3/assignment_loss‚0.2172303041166711 0loss/FRt4/assignment_loss‚2.8630882183710735 1loss/SRqqt3/assignment_loss‚1.062575309283354 /loss/FRt4/detection_loss‚2.4437388519446057 /loss/FBt1/detection_loss‚2.1780409465233483 epoch‚5 _runtime‚5127.353101028 1loss/SRqqt2/detection_loss‚1.8497322841036705 1loss/SRqqt3/detection_loss‚1.1429439543774633 .loss/FBt2/detection_loss‚1.775650476415952 /loss/FRt1/assignment_loss‚4.812425480948554 1loss/SRqqt2/assignment_loss‚1.381329893074449 0loss/FBt4/assignment_loss‚0.4586617611344839 1loss/SRqqt1/detection_loss‚2.7574049631754556 2loss/SRqqt4/assignment_loss‚1.8726518608274914 0loss/FBt1/assignment_loss‚0.6151587087551889 .loss/FRt2/detection_loss‚2.280585476756096 .loss/FRt3/detection_loss‚1.997559439142545 trainer/global_step‚6449’²3n“Ø‚(:ˆ Þ©æÔÖÁƒý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.082' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2164 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚06§¶‡tÙ‚(:é Þ©æÔ¿°—€( disk.nvme0c0n1.out‚20756.52490234375& proc.memory.availableMB‚ 957.171875 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.out‚ 198.30859375 disk.drbd1716.in‚0 network.recv‚ 1.859875e+06 disk.sda1.in‚0 network.sent‚ 1.529332e+07# disk.dm-0.out‚20756.52490234375 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.rbd0.in‚0 cpu‚25.344736643011146* disk./.usagePercent‚20.243022627297595 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 27.841796875 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚20756.52490234375 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda.out‚ 198.30859375 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 198.30859375 disk.rbd0.out‚0$ memory_percent‚94.15788650512695" proc.memory.rssMB‚ 2621.6484375 proc.cpu.threads‚17* proc.memory.percent‚16.001272201538086$ disk./.usageGB‚301.6356658935547 disk.dm-0.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.drbd2080.out‚0N*ñžÚ‚(:“ í©æÔ›š–ý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚188.731' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.746199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0NüfgÛ‚(:Ü í©æÔÌÓ‰ÿ# memory_percent‚94.1563606262207 network.recv‚ 1.865469e+06 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 198.90625$ disk./.usageGB‚301.6207084655762 disk.rbd0.in‚0 cpu‚25.343845103999257) disk./.usagePercent‚20.24201882175584 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 disk.sda1.in‚0 disk.sda2.in‚0% disk.nvme0n1.out‚20801.3408203125" disk.dm-0.out‚20801.3408203125 disk.drbd2266.out‚0 disk.dm-1.out‚ 198.90625 disk.sda.out‚ 198.90625 disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875& proc.memory.availableMB‚ 957.421875 disk.rbd1.out‚ 27.904296875 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2621.6484375* proc.memory.percent‚16.001272201538086 proc.cpu.threads‚17 network.sent‚ 1.5401156e+07 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚20801.3408203125 disk.dm-0.in‚0.078125ÒzôÄ“Ü‚(:ˆ ü©æÔðëþü gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.654' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9308 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0BPð±ºÝ‚(:â ü©æÔ³³òÿ network.sent‚ 1.5407688e+07) disk./.usagePercent‚20.24385567581401 disk.sda.out‚ 199.484375 disk.rbd1.out‚ 27.904296875 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0( proc.memory.availableMB‚ 957.16796875" proc.memory.rssMB‚ 2621.6484375 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.in‚0% disk.nvme0n1.out‚20839.0400390625 disk.dm-1.out‚ 199.484375 disk.drbd2265.in‚0* proc.memory.percent‚16.001272201538086 disk.dm-0.in‚0.078125" disk.dm-0.out‚20839.0400390625 disk.rbd1.in‚0' disk.nvme0c0n1.out‚20839.0400390625 disk.sda3.out‚ 199.484375% disk./.usageGB‚301.64807891845703 disk.drbd2266.out‚0 network.recv‚ 1.87047e+06 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0$ memory_percent‚94.15791034698486 proc.cpu.threads‚17êoz¿³ disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.drbd2265.out‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0 cpu‚25.342987478382796ØëÐ2Þ- _wandbruntime‚5163  _runtime‚5163ù7ÇÇßÁ & lr-AdamW/pg1‚6.609440303772125e-05 & lr-AdamW/pg2‚6.609440303772125e-05 trainer/global_step‚6499 % _timestamp‚1.7884500478168936e+09 _runtime‚5163.288538715 _step‚531“hÓEÌàÆ trainer/global_step‚6499 % _timestamp‚1.7884500478168936e+09 _runtime‚5163.288538715 _step‚531 & lr-AdamW/pg1‚6.609440303772125e-05 & lr-AdamW/pg2‚6.609440303772125e-05“>×2á- _wandbruntime‚5163  _runtime‚5163Q(¹à'âÚO ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚418391325721600 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚94515486 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚377913106066944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5443660054016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚94515488 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚466 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5710219687424 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 122690989 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚383356784123904 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚368010066267136 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 $ _timestamp‚1.788450047817608e+09 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚373433702276050 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚368010083306496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚6499 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 122711027 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176206848 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚94515488 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 122711027 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚55146462 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚383356109758464 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200146944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚418391525868544 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 _step‚532 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚377913106066944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31103877 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5423618969554 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 217226515 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚383356109758464 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 122690989 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5710043480576 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚373433030798934 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5443003691520 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5422964531798Q9<å'ãßO N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 217226515 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚383356109758464 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 122690989 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5710043480576 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚373433030798934 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5443003691520 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5422964531798 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚418391325721600 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚94515486 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚377913106066944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5443660054016 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚94515488 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚466 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5710219687424 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 122690989 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚383356784123904 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚368010066267136 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 $ _timestamp‚1.788450047817608e+09 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚373433702276050 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚368010083306496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 trainer/global_step‚6499 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 122711027 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176206848 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚94515488 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 122711027 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚55146462 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚383356109758464 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200146944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚418391525868544 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 _step‚532 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚377913106066944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31103877 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5423618969554 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952”¢|Æ2ä- _wandbruntime‚5163  _runtime‚5163bÅøë&ååM ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚368064987437056 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚94529502 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31108971 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚418454033107968 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5423780966762 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚377969459105792 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 217259011 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚383413956603904 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚418453853945856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 122709443 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚86263151 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚377969459105792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚55154630 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 % _timestamp‚1.7884500485047991e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚55154628 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚377969477108736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31108523 _step‚533 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚94529504 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5443823119360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _runtime‚5163.977097311 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 217238945 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 122729507 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚383413956603904 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚94529502 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5711076189184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 122709443 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 217259011 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚86263151 ZFDeviceStatsMonitor.on_train_batch_end/allocaË—ª¼Â ted_bytes.large_pool.freed‚377969459105792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚55154630 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 % _timestamp‚1.7884500485047991e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚55154628 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚377969477108736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31108523 _step‚533 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚94529504 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5443823119360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _runtime‚5163.977097311 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 217238945 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 122729507 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚383413956603904 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚94529502 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5711076189184 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 122709443 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 217259011 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5423780966762 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚377969459105792 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 217259011 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚383413956603904 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚418453853945856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 122709443 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5753136043520 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚380730758009344 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5483989497344 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚376219744143458 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚386214747506688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5483989497344 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚55554860 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚376219072666342 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚370755289834496 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚55554862 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚437 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚421516979107840 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 218853367 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5752964030976 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚95216288 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5463799871206 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚95216288 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31330611 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚6549 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 218833327 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 123617041 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31330174 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _step‚536 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 123617041 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚415763843064320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5464454308962 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚86885473 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _runtime‚5200.072632859 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚95216286 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚421516783155200 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5753136043520 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚380730758009344 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5483989497344 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚376219744143458 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚386214747506688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5483989497344 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚55554860 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚376219072666342 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚370755289834496 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚55554862 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚437 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚421516979107840 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 218853367 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5752964030976 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚95216288 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5463799871206 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚95216288 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31330611 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚6549 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 218833327 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 123617041 TGDeviceSta),†´Õ tsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31330174 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _step‚536 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 123617041 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚415763843064320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5464454308962 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚86885473 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _runtime‚5200.072632859 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚95216286 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚421516783155200 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 218885863 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5484808925184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚380787129051136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 218865797 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚421579485725696 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚415825493801984 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 123635495 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5465270744134 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚386272594352128 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚370810176925696 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 218865797 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚386271919973376 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚95230304 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚95230302 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 _step‚537 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚415825469861888 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 123655559 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚421579298174976 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 123655559 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 123635495 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚376275464709190 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5464616306170 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5485465300992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884500853010328e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚55563028 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚386272594352128 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚380787129051136 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚380787111048192 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚95230304 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚86898079 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚95230302 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚376274793231866 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚55563030 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 218885863 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚86897644 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5753991923712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414¬¹ìäð&öêM JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚5200.773217784 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚86898079 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚95230302 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚376274793231866 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚55563030 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 218885863 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚86897644 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5753991923712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5753828313088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5484808925184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚380787111048192 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 218885863 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5484808925184 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚380787129051136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 218865797 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚421579485725696 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚415825493801984 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 123635495 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5465270744134 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚386272594352128 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚370810176925696 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 218865797 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚386271919973376 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚95230304 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚95230302 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 _step‚537 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚415825469861888 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 123655559 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚421579298174976 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 123655559 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 123635495 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚376275464709190 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5464616306170 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5485465300992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884500853010328e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚55563028 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚386272594352128 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚380787129051136 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚380787111048192 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚95230304 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BrS2€- _wandbruntime‚5236  _runtime‚5236šºÇÁ & lr-AdamW/pg1‚6.426638407521579e-05 & lr-AdamW/pg2‚6.426638407521579e-05 trainer/global_step‚6599 % _timestamp‚1.7884501214962397e+09 _runtime‚5236.967835001 _step‚539=X4Ì‚Æ _step‚539 & lr-AdamW/pg1‚6.426638407521579e-05 & lr-AdamW/pg2‚6.426638407521579e-05 trainer/global_step‚6599 % _timestamp‚1.7884501214962397e+09 _runtime‚5236.967835001›(Ië2ƒ- _wandbruntime‚5236  _runtime‚5236²ÐR.¡„ÚO ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚424642459100672 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31558650 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚5236.968744808 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚95917088 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚95917086 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 124563131 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚379005114533750 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚95917086 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚389074059620352 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚389073385254912 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5524975303168 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 220480219 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 124543093 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31559101 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5525631665664 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5795919723520 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 124563131 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚389074059620352 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 220460179 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚383548409951744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚95917088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚424642275730944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5796079153152 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5524975303168 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5505289648370 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚87521910 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 _step‚540 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚418846379947520 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 $ _timestamp‚1.788450121496948e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5525631665664 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚373500479323136 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚389073385254912 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 124543093 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚383548427954688 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚418846356007424 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚55963260 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5504635210614· ¾å'…ßO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚389073385254912 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5524975303168 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 220480219 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 124543093 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31559101 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5525631665664 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5795919723520 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 124563131 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚389074059620352 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 220460179 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚383548409951744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚95917088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚424642275730944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5796079153152 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5524975303168 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5505289648370 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚87521910 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 _step‚540 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚418846379947520 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 $ _timestamp‚1.788450121496948e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5525631665664 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚373500479323136 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚389073385254912 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 124543093 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚383548427954688 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚418846356007424 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚55963260 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5504635210614 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚424642459100672 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31558650 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚5236.968744808 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚95917088 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚95917086 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 124563131 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚379005114533750 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚95917086 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚389074059620352 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚87534624 _runtime‚5237.655201945 _step‚541 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 124561547 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 trainer/global_step‚6599 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 220492649 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5525794731008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚55971430 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚424704782787072 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚379060835099274 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5526451106816 $ _timestamp‚1.788450122183077e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚95931104 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5506106083542 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 124581611 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚383604762990592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31563196 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚95931102 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚424704972434944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5526451106816 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚389130557721600 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚95931102 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 220512715 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 220512715 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚95931104 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚55971428 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚383604762990592 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31563647 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚373555400493056 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚383604780993536 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5505451645578 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808KR ˆéM XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚424704782787072 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚379060835099274 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5526451106816 $ _timestamp‚1.788450122183077e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚95931104 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5506106083542 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 124581611 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚383604762990592 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31563196 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚95931102 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚424704972434944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5526451106816 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚389130557721600 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚95931102 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 220512715 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 220512715 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚95931104 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚55971428 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚383604762990592 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31563647 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚373555400493056 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated¼[{R‚ 868220928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚383604780993536 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5505451645578 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 220492649 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚418908030685184 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚418908006745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚87534624 _runtime‚5237.655201945 _step‚541 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 124561547 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 trainer/global_step‚6599 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 220492649 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5525794731008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚55971430 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889w­o2‰- _wandbruntime‚5237  _runtime‚5237¿½Œ•XŠÒ epoch‚5 .loss/FBt2/detection_loss‚2.615799318660389 .loss/FRt1/assignment_loss‚5.45031034151713 0loss/FRt3/assignment_loss‚1.8320569784744927 _runtime‚5237.656249552 2loss/SRqqt1/assignment_loss‚2.6484631774602114 2loss/SRqqt2/assignment_loss‚1.6571275629103184 1loss/SRqqt3/assignment_loss‚1.290454618819058 1loss/FBt2/assignment_loss‚0.12181713452628173 .loss/FRt2/detection_loss‚2.086069209575653 /loss/FRt3/detection_loss‚2.1096988070108322 1loss/SRqqt1/detection_loss‚2.5804357837747642 .loss/FBt1/detection_loss‚2.633371396859487 &loss/total_loss‚47.256964937971595 .loss/FRt2/assignment_loss‚3.21087362408638 0loss/FBt1/assignment_loss‚0.3066902994598877 1loss/FBt3/assignment_loss‚0.21081672964919873 1loss/FBt4/assignment_loss‚0.08472528675119975 1loss/SRqqt2/detection_loss‚1.6225775803985267 /loss/FBt3/detection_loss‚0.9780574236574466 1loss/SRqqt4/assignment_loss‚1.144447242180186 trainer/global_step‚6599 _step‚542 0loss/FRt4/assignment_loss‚2.4700817559894763 /loss/FRt4/detection_loss‚2.0785878360794303 1loss/SRqqt3/detection_loss‚1.4385364834219216 1loss/SRqqt4/detection_loss‚1.4162014782909425 .loss/FBt4/detection_loss‚4.160292184352874 % _timestamp‚1.7884501221847398e+09 .loss/FRt1/detection_loss‚3.109472684065501鹸{]‹× 1loss/SRqqt2/detection_loss‚1.6225775803985267 /loss/FBt3/detection_loss‚0.9780574236574466 1loss/SRqqt4/assignment_loss‚1.144447242180186 trainer/global_step‚6599 _step‚542 0loss/FRt4/assignment_loss‚2.4700817559894763 /loss/FRt4/detection_loss‚2.0785878360794303 1loss/SRqqt3/detection_loss‚1.4385364834219216 1loss/SRqqt4/detection_loss‚1.4162014782909425 .loss/FBt4/detection_loss‚4.160292184352874 % _timestamp‚1.7884501221847398e+09 .loss/FRt1/detection_loss‚3.109472684065501 epoch‚5 .loss/FBt2/detection_loss‚2.615799318660389 .loss/FRt1/assignment_loss‚5.45031034151713 0loss/FRt3/assignment_loss‚1.8320569784744927 _runtime‚5237.656249552 2loss/SRqqt1/assignment_loss‚2.6484631774602114 2loss/SRqqt2/assignment_loss‚1.6571275629103184 1loss/SRqqt3/assignment_loss‚1.290454618819058 1loss/FBt2/assignment_loss‚0.12181713452628173 .loss/FRt2/detection_loss‚2.086069209575653 /loss/FRt3/detection_loss‚2.1096988070108322 1loss/SRqqt1/detection_loss‚2.5804357837747642 .loss/FBt1/detection_loss‚2.633371396859487 &loss/total_loss‚47.256964937971595 .loss/FRt2/assignment_loss‚3.21087362408638 0loss/FBt1/assignment_loss‚0.3066902994598877 1loss/FBt3/assignment_loss‚0.21081672964919873 1loss/FBt4/assignment_loss‚0.08472528675119975ž»Œ“Œ‚(:ˆ ÖªæÔÄÕúü gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.379' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2758 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0z9ˆNi‚(:Þ ÖªæÔéÀÿ disk.sda3.out‚ 202.78125 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 network.sent‚ 1.5648584e+07 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 cpu‚25.337952832310226 proc.cpu.threads‚17 disk.drbd2080.in‚0# disk.dm-0.out‚21113.42822265625& proc.memory.availableMB‚ 957.234375 network.recv‚ 1.900684e+06" proc.memory.rssMB‚ 2621.6484375 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 28.453125$ disk./.usageGB‚301.6789970397949 disk.dm-1.out‚ 202.78125 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0& disk.nvme0n1.out‚21113.42822265625* proc.memory.percent‚16.001272201538086 disk.drbd1716.out‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚21113.42822265625 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda2.out‚0$ memory_percent‚94.15750503540039 disk.sda.out‚ 202.78125 disk.sda1.in‚0 disk.sda1.out‚0* disk./.usagePercent‚20.245930616879033 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2265.in‚0…ŃžŽ‚(:“ åªæÔõ€ý gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚145.523' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.104599999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0](NÖj‚(:ß åªæÔ˾ˆÿ disk.rbd1.out‚ 28.515625 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 network.recv‚ 1.906063e+06 disk.nvme0n1.in‚0.078125% disk.nvme0c0n1.out‚21155.06640625 disk.rbd0.in‚0 proc.cpu.threads‚17 network.sent‚ 1.5756345e+07 disk.dm-0.in‚0.078125 disk.sda2.out‚0" proc.memory.rssMB‚ 2621.6484375* proc.memory.percent‚16.001272201538086# disk.nvme0n1.out‚21155.06640625 disk.drbd2266.out‚0 disk.dm-0.out‚21155.06640625 disk.dm-1.out‚ 203.30078125 disk.sda.in‚ 0.01171875 disk.sda.out‚ 203.30078125 disk.sda1.in‚0( proc.memory.availableMB‚ 957.23046875 cpu‚25.337137610318862 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.sda1.out‚0) disk./.usagePercent‚20.24644724008313 disk.sda3.out‚ 203.30078125 disk.sda2.in‚0# memory_percent‚94.1575288772583% disk./.usageGB‚301.68669509887695 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0¸æ2- _wandbruntime‚5273  _runtime‚5273õ†µAÅ‘¿ % _timestamp‚1.7884501582100763e+09 _runtime‚5273.681656467 _step‚543 % lr-AdamW/pg1‚6.33519906390989e-05 % lr-AdamW/pg2‚6.33519906390989e-05 trainer/global_step‚66492ÂØÊ’Ä trainer/global_step‚6649 % _timestamp‚1.7884501582100763e+09 _runtime‚5273.681656467 _step‚543 % lr-AdamW/pg1‚6.33519906390989e-05 % lr-AdamW/pg2‚6.33519906390989e-05Ÿzp!¬2“- _wandbruntime‚5273  _runtime‚5273ÈÐcŽá'”ÛO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 222087031 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5565961108992 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 125489183 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚376245702890496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚391932697368576 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚386366079897088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚421928916830720 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚386366061894144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5565961108992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚56371662 trainer/global_step‚6649 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31786196 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5839034174976 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚386366079897088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚96617888 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5545470550022 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚56371660 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚391932023003136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚391932697368576 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 125469145 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31785760 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚427767763441664 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚96617888 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5566617471488 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚421928892890624 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚381791156401158 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚544 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚386366061894144 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚88157420 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚427767951005696áZe×æ'•àO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚391932697368576 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚386366079897088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚421928916830720 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚386366061894144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5565961108992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚56371662 trainer/global_step‚6649 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚31786196 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5839034174976 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚386366079897088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚96617888 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5545470550022 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚56371660 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚391932023003136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚391932697368576 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 125469145 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚31785760 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚427767763441664 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚96617888 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5566617471488 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚421928892890624 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚381791156401158 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚544 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚386366061894144 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚88157420 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚427767951005696 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 222087031 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5565961108992 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 125489183 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚376245702890496 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5839896762368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 125507663 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚376300607021056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚386422432935936 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚96631904 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 125487599 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚391989195469824 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚88170066 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚423 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚386422432935936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚88170491 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚381847548444006 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 125507663 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31790238 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 222139567 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884501588958707e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚391989869848576 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚386422414932992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚56379830 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚427830464330752 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚5274.367859079 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚391989869848576 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚425 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5546286984986 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚427830278877184 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 222119501 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚56379828 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31790661 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚381846876966682 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 222139567 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚386422414932992 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 125487599 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5839735248896 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430ˆnÓÂð&˜êM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚391989869848576 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚425 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5546286984986 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚427830278877184 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 222119501 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚56379828 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚31790661 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚381846876966682 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 222139567 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚386422414932992 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 125487599 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5839735248896 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5566780536832 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5567436912640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚96631902 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5839896762368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 125507663 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚376300607021056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚386422432935936 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚96631904 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 125487599 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚391989195469824 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚88170066 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚423 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚386422432935936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚88170491 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚381847548444006 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 125507663 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚31790238 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 222139567 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884501588958707e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚391989869848576 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚386422414932992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚56379830 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚427830464330752 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚5274.367859079¡3v'·2™- _wandbruntime‚5274  _runtime‚5274Ñ,qXšÒ /loss/FRt4/detection_loss‚2.2622487198176056 /loss/SRqqt1/detection_loss‚2.53477143868804 _runtime‚5274.368977029 _step‚546 2loss/SRqqt1/assignment_loss‚1.9183254558593035 0loss/FBt2/assignment_loss‚0.4427686199562659 1loss/FBt3/assignment_loss‚0.25952600512750434 0loss/FBt4/assignment_loss‚1.3812293531373143 /loss/FBt3/detection_loss‚0.7274625034304336 /loss/FRt1/assignment_loss‚4.720939453442892 .loss/FRt1/detection_loss‚3.087534707784653 0loss/SRqqt2/detection_loss‚1.863182070851326 /loss/FBt2/detection_loss‚2.1509800269490196 % _timestamp‚1.7884501588974597e+09 0loss/FRt3/assignment_loss‚1.8436037947734196 .loss/FRt2/detection_loss‚2.038190670311451 %loss/total_loss‚48.81723650429892 epoch‚5 trainer/global_step‚6649 2loss/SRqqt4/assignment_loss‚1.4568424985211874 0loss/FRt2/assignment_loss‚3.6110412863393626 2loss/SRqqt2/assignment_loss‚1.4389616173878312 /loss/FRt3/detection_loss‚1.6658639170229435 1loss/SRqqt4/detection_loss‚1.8366855782817821 /loss/FBt1/detection_loss‚2.5239761739969255 /loss/FRt4/assignment_loss‚3.423001837113808 2loss/SRqqt3/assignment_loss‚1.2420779874016132 0loss/FBt1/assignment_loss‚0.6384870320250229 1loss/SRqqt3/detection_loss‚1.2992652821221522 .loss/FBt4/detection_loss‚4.450270473957062šªÍ]›× 0loss/FBt4/assignment_loss‚1.3812293531373143 /loss/FBt3/detection_loss‚0.7274625034304336 /loss/FRt1/assignment_loss‚4.720939453442892 .loss/FRt1/detection_loss‚3.087534707784653 0loss/SRqqt2/detection_loss‚1.863182070851326 /loss/FBt2/detection_loss‚2.1509800269490196 % _timestamp‚1.7884501588974597e+09 0loss/FRt3/assignment_loss‚1.8436037947734196 .loss/FRt2/detection_loss‚2.038190670311451 %loss/total_loss‚48.81723650429892 epoch‚5 trainer/global_step‚6649 2loss/SRqqt4/assignment_loss‚1.4568424985211874 0loss/FRt2/assignment_loss‚3.6110412863393626 2loss/SRqqt2/assignment_loss‚1.4389616173878312 /loss/FRt3/detection_loss‚1.6658639170229435 1loss/SRqqt4/detection_loss‚1.8366855782817821 /loss/FBt1/detection_loss‚2.5239761739969255 /loss/FRt4/assignment_loss‚3.423001837113808 2loss/SRqqt3/assignment_loss‚1.2420779874016132 0loss/FBt1/assignment_loss‚0.6384870320250229 1loss/SRqqt3/detection_loss‚1.2992652821221522 .loss/FBt4/detection_loss‚4.450270473957062 /loss/FRt4/detection_loss‚2.2622487198176056 /loss/SRqqt1/detection_loss‚2.53477143868804 _runtime‚5274.368977029 _step‚546 2loss/SRqqt1/assignment_loss‚1.9183254558593035 0loss/FBt2/assignment_loss‚0.4427686199562659 1loss/FBt3/assignment_loss‚0.25952600512750434¢ê~“œ‚(:ˆ ôªæÔœ¾„ý gpu.0.gpu‚44.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚144.044' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚28.8088 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ùÖ˜ád‚(:Ù ôªæÔ¸«ÿ network.sent‚ 1.5762763e+07 network.recv‚ 1.91098e+06 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.drbd1716.out‚0& proc.memory.availableMB‚ 957.203125 disk.rbd0.in‚0 disk.sda.out‚ 203.87890625 disk.sda1.in‚0 disk.sda2.out‚0# disk.nvme0n1.out‚21196.50390625% disk.nvme0c0n1.out‚21196.50390625 disk.dm-0.out‚21196.50390625 disk.sda1.out‚0 disk.dm-1.out‚ 203.87890625 disk.drbd1716.in‚0 cpu‚25.336323632678 disk.drbd2080.in‚0 disk.sda3.out‚ 203.87890625$ memory_percent‚94.15769577026367* proc.memory.percent‚16.001272201538086 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.nvme0n1.in‚0.078125" proc.memory.rssMB‚ 2621.6484375 disk.drbd2080.out‚0 disk.rbd1.out‚ 28.609375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0( disk./.usagePercent‚20.2465337706297% disk./.usageGB‚301.68798446655273 proc.cpu.threads‚17 disk.dm-1.in‚ 0.01171875ú±åd“ž‚(:ˆ ƒ«æÔåå¤ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.192' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.8384 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0( é¿nŸ‚(:ã ƒ«æÔ›¾­ÿ disk.sda3.out‚ 204.3203125 disk.sda2.out‚0 network.sent‚ 1.5870824e+07 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 204.3203125 disk.rbd1.out‚ 28.796875* disk./.usagePercent‚20.247127452101605$ disk./.usageGB‚301.6968307495117 disk.rbd0.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0& proc.memory.availableMB‚ 957.140625" proc.memory.rssMB‚ 2621.6484375 cpu‚25.33551679601129 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0# disk.dm-0.out‚21231.12744140625 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0$ memory_percent‚94.15807723999023 proc.cpu.threads‚17 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.dm-1.out‚ 204.3203125 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚21231.12744140625 disk.drbd2265.out‚0* proc.memory.percent‚16.001272201538086 network.recv‚ 1.916704e+06 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚21231.12744140625$U˜“ ‚(:ˆ ’«æÔ‡óý gpu.0.gpu‚43.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚190.019' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.0038 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¶o¥q¡‚(:æ ’«æԚ˞ÿ disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 204.90234375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.rbd1.out‚ 28.796875$ disk./.usageGB‚301.6662368774414# disk.dm-0.out‚21287.31494140625 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.rbd0.out‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 204.90234375 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0$ memory_percent‚94.15807723999023& proc.memory.availableMB‚ 957.140625 network.sent‚ 1.587744e+07 disk.sda1.out‚0 disk.drbd1716.in‚0* disk./.usagePercent‚20.245074271676998 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda2.in‚0& disk.nvme0n1.out‚21287.31494140625 network.recv‚ 1.921574e+06 disk.drbd2265.in‚0* proc.memory.percent‚16.001272201538086 cpu‚25.334715840147684 disk.sda2.out‚0 disk.sda.out‚ 204.90234375( disk.nvme0c0n1.out‚21287.31494140625" proc.memory.rssMB‚ 2621.6484375 disk.dm-0.in‚0.078125håÑ2¢- _wandbruntime‚5310  _runtime‚5310¡Gq£Ç£Á _runtime‚5310.541386542 _step‚547 & lr-AdamW/pg1‚6.243759529733162e-05 & lr-AdamW/pg2‚6.243759529733162e-05 trainer/global_step‚6699 % _timestamp‚1.7884501950697777e+098‘ÐÌ¤Æ % _timestamp‚1.7884501950697777e+09 _runtime‚5310.541386542 _step‚547 & lr-AdamW/pg1‚6.243759529733162e-05 & lr-AdamW/pg2‚6.243759529733162e-05 trainer/global_step‚6699£“LË2¥- _wandbruntime‚5310  _runtime‚5310BÉ4B¦ÛO YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚389183731839488 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 223733923 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚389183713836544 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚6699 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5606946914816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚394791335116800 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚389183713836544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚389183731839488 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚97318688 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 126395197 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚394791335116800 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 126395197 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5606946914816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚97318688 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚32012304 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5607603277312 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚548 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚394790660751360 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚88792807 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚56780062 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _runtime‚5310.542297541 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 126415235 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5881926160896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚384577869745682 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚441 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚394790660751360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚430893547693056 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚32012745 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884501950705085e+09 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚56780060 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 126415235 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚378990909418496 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚97318686 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚430893355934720 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚384577198268566 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚97318686 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5586960327186 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚88792364 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908kËs‡æ'§àO D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚389183731839488 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚97318688 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 126395197 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚394791335116800 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 126395197 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5606946914816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚97318688 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚32012304 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5607603277312 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚548 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚394790660751360 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚88792807 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚56780062 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 _runtime‚5310.542297541 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 126415235 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5881926160896 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚384577869745682 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚441 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚394790660751360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚430893547693056 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚32012745 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884501950705085e+09 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚56780060 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 126415235 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚378990909418496 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚97318686 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚430893355934720 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚384577198268566 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚97318686 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5586960327186 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚88792364 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚389183731839488 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 223733923 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚389183713836544 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚6699 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5606946914816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚394791335116800 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚389183713836544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 223766419 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 223746353 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5587776762358 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 126433715 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚389240066875392 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚394848507596800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32017284 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚88805514 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚425073080511488 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚430956056162304 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚379045796509696 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚389240084878336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5882951710720 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5607766342656 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚56788230 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5587122324394 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 126433715 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚389240066875392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚97332702 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 126413651 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚97332702 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 223746353 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚389240084878336 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7884501957605977e+09 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚56788228 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5882790197248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚97332704 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚88805064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 126413651 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚394847833218048 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 223766419 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚384632918834090 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430³ðµ®;ªêM UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚389240066875392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚97332702 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 126413651 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚97332702 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 223746353 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚389240084878336 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7884501957605977e+09 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚56788228 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5882790197248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚97332704 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚88805064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 126413651 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚394847833218048 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 223766419 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚384632918834090 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚384633590311414 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _runtime‚5311.232636074 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚97332704 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚32016836 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 223766419 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 223746353 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5587776762358 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 126433715 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚389240066875392 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚394848507596800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32017284 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.fYùË0µ reed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚88805514 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚425073080511488 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚430956056162304 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚379045796509696 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚389240084878336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5882951710720 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5607766342656 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚56788230 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5587122324394 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 126433715 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025¥µÄ:í2«- _wandbruntime‚5311  _runtime‚5311 æCªX¬Ò 2loss/SRqqt1/assignment_loss‚1.7068090751767158 2loss/SRqqt3/assignment_loss‚1.4930381487443376 2loss/SRqqt4/assignment_loss‚1.5687554852794046 /loss/FRt1/assignment_loss‚4.312425189548069 0loss/FBt1/assignment_loss‚0.6295398715881788 0loss/SRqqt1/detection_loss‚2.938812429904938 /loss/FBt3/detection_loss‚0.6564725529311634 _step‚550 0loss/FBt3/assignment_loss‚0.1711128341993612 /loss/FRt4/detection_loss‚2.2484918115315615 %loss/total_loss‚46.49825666547807 0loss/FRt3/assignment_loss‚1.9337193810939788 /loss/FBt2/assignment_loss‚0.580404421227243 1loss/FBt4/assignment_loss‚0.33524684011915723 /loss/FRt3/detection_loss‚1.8849079900979995 /loss/FBt1/detection_loss‚2.5415355265140533 /loss/FBt4/detection_loss‚2.9946011304855347 % _timestamp‚1.7884501957622242e+09 /loss/FRt2/assignment_loss‚3.561641310868056 2loss/SRqqt2/assignment_loss‚1.2780729367990384 /loss/FRt1/detection_loss‚3.5392108625835843 .loss/FBt2/detection_loss‚2.003065991215408 /loss/FRt2/detection_loss‚2.0509181320667267 /loss/FRt4/assignment_loss‚3.232013848092821 0loss/SRqqt3/detection_loss‚1.239305365219045 epoch‚5 trainer/global_step‚6699 _runtime‚5311.233706264 0loss/SRqqt2/detection_loss‚1.903621031479402 1loss/SRqqt4/detection_loss‚1.6945344987122908ÂûÑ]­× 0loss/FRt3/assignment_loss‚1.9337193810939788 /loss/FBt2/assignment_loss‚0.580404421227243 1loss/FBt4/assignment_loss‚0.33524684011915723 /loss/FRt3/detection_loss‚1.8849079900979995 /loss/FBt1/detection_loss‚2.5415355265140533 /loss/FBt4/detection_loss‚2.9946011304855347 % _timestamp‚1.7884501957622242e+09 /loss/FRt2/assignment_loss‚3.561641310868056 2loss/SRqqt2/assignment_loss‚1.2780729367990384 /loss/FRt1/detection_loss‚3.5392108625835843 .loss/FBt2/detection_loss‚2.003065991215408 /loss/FRt2/detection_loss‚2.0509181320667267 /loss/FRt4/assignment_loss‚3.232013848092821 0loss/SRqqt3/detection_loss‚1.239305365219045 epoch‚5 trainer/global_step‚6699 _runtime‚5311.233706264 0loss/SRqqt2/detection_loss‚1.903621031479402 1loss/SRqqt4/detection_loss‚1.6945344987122908 2loss/SRqqt1/assignment_loss‚1.7068090751767158 2loss/SRqqt3/assignment_loss‚1.4930381487443376 2loss/SRqqt4/assignment_loss‚1.5687554852794046 /loss/FRt1/assignment_loss‚4.312425189548069 0loss/FBt1/assignment_loss‚0.6295398715881788 0loss/SRqqt1/detection_loss‚2.938812429904938 /loss/FBt3/detection_loss‚0.6564725529311634 _step‚550 0loss/FBt3/assignment_loss‚0.1711128341993612 /loss/FRt4/detection_loss‚2.2484918115315615 %loss/total_loss‚46.49825666547807¦õÕ¼”“®‚(:ˆ ¡«æԙ̃ý gpu.0.gpu‚30.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚174.048' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.8096 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0FÞý‹d¯‚(:Ù ¡«æÔÖ„¯€ disk.drbd2266.out‚0 disk.dm-0.out‚ 21345.0390625 disk.rbd1.in‚0 disk.drbd1716.out‚0* proc.memory.percent‚16.001272201538086 disk.sda.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875# memory_percent‚94.1584587097168" proc.memory.rssMB‚ 2621.6484375 cpu‚25.33390699022259 proc.cpu.threads‚17% disk./.usageGB‚301.73754501342773 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 network.recv‚ 1.925195e+06) disk./.usagePercent‚20.24985982051455 disk.rbd0.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 28.859375 disk.dm-1.out‚ 205.38671875 disk.drbd2265.out‚0 network.sent‚ 1.5882684e+07 disk.rbd0.in‚0 disk.sda1.out‚0" disk.nvme0n1.out‚ 21345.0390625$ disk.nvme0c0n1.out‚ 21345.0390625& proc.memory.availableMB‚ 957.078125 disk.sda.out‚ 205.38671875 disk.sda1.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 205.38671875 disk.dm-0.in‚0.078125 disk.drbd1716.in‚0x˜CÍ“°‚(:ˆ °«æÔþƒý gpu.0.gpu‚55.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚126.466' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.2932 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0j†\b±‚(:× °«æÔêë£ÿ disk.sda2.out‚0" disk.nvme0n1.out‚ 21404.6328125 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 206.03515625 disk.rbd0.in‚0" proc.memory.rssMB‚ 2621.6484375 cpu‚25.3331712366533 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0 disk.drbd2266.in‚0 network.sent‚ 1.5991731e+07 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.out‚0# memory_percent‚94.1584587097168 disk.drbd2265.in‚0 disk.dm-0.out‚ 21404.6328125 disk.sda1.in‚0& proc.memory.availableMB‚ 957.078125* proc.memory.percent‚16.001272201538086 proc.cpu.threads‚17 network.recv‚ 1.932232e+06) disk./.usagePercent‚20.25170256274601 disk.rbd1.in‚0 disk.rbd1.out‚ 28.859375 disk.dm-1.out‚ 206.03515625 disk.drbd1716.out‚0 disk.drbd2265.out‚0$ disk.nvme0c0n1.out‚ 21404.6328125$ disk./.usageGB‚301.7650032043457 disk.dm-0.in‚0.078125 disk.sda.out‚ 206.03515625E.Æ2²- _wandbruntime‚5347  _runtime‚5347`ã  Æ³À & lr-AdamW/pg2‚6.152338860853263e-05 trainer/global_step‚6749 $ _timestamp‚1.788450231760447e+09 _runtime‚5347.232067439 _step‚551 & lr-AdamW/pg1‚6.152338860853263e-05¶‚çË´Å & lr-AdamW/pg1‚6.152338860853263e-05 & lr-AdamW/pg2‚6.152338860853263e-05 trainer/global_step‚6749 $ _timestamp‚1.788450231760447e+09 _runtime‚5347.232067439 _step‚551§¾ÐDÜ2µ- _wandbruntime‚5347  _runtime‚5347Rí¢îá'¶ÛO =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚397649972865024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚428093966657024 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚392001383781888 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 225360775 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚387363240135974 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚32239696 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5648589083136 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5648589083136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚98019486 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 % _timestamp‚1.7884502317612495e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚89427714 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚57188460 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚98019488 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5627141228838 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚392001365778944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚98019488 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5627795666594 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 127321249 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 225340735 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚397649298499584 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚89428158 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚6749 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚392001383781888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 127321249 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5924932297216 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚381736098907136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 225340735 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚397649298499584 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚89428158 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚6749 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚392001383781888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 127321249 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5924932297216 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚381736098907136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚397649972865024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚428093966657024 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚392001383781888 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 225360775 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚387363240135974 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚32239696 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5648589083136 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5648589083136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚98019486 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 % _timestamp‚1.7884502317612495e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚89427714 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚57188460 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚98019488 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5627141228838 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚392001365778944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚98019488 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5627795666594 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 127321249 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5649408524288 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚397707145345024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚98033504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5628612101766 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 127339703 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚6749 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5925792755712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 % _timestamp‚1.7884502324455478e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚89440378 XDDeviceStatsMonitor.µQ$Ñon_train_batch_end/inactive_split_bytes.all.freed‚434081410150400 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚89440811 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 225373205 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5648752148480 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚428155641334784 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚98033504 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚57196628 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 127339703 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚98033502 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚57196630 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚381791020077056 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚5347.917630302 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32244181 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚397706470966272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 225393271 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚392057736820736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚428155617394688 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5925954269184 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚392057718817792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚392057718817792 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5627957663802 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 225393271 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 225373205 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚433 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5648752148480 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚381791003037696wkñvð&ºêM I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚6749 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5925792755712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 % _timestamp‚1.7884502324455478e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚89440378 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚434081410150400 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚89440811 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 225373205 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5648752148480 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚428155641334784 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚98033504 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚57196628 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 127339703 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚98033502 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚57196630 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚381791020077056 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚5347.917630302 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32244181 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚397706470966272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 225393271 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚392057736820736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚428155617394688 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5925954269184 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚392057718817792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚392057718817792 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5627957663802 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 225393271 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 225373205 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚433 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5648752148480 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚381791003037696 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚397707145345024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5649408524288 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚397707145345024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚98033504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5628612101766 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 127339703 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521©HS‘è2»- _wandbruntime‚5347  _runtime‚5347ä"%W¼Ñ /loss/FRt2/assignment_loss‚3.427400556632451 1loss/FBt2/assignment_loss‚0.31998274330410936 0loss/FBt4/assignment_loss‚0.9404360577462967 0loss/SRqqt4/detection_loss‚1.952470481881629 0loss/FBt3/assignment_loss‚0.2137387490003521 /loss/FRt1/detection_loss‚2.9986498022079466 /loss/FRt2/detection_loss‚1.7352681194032942 1loss/SRqqt2/detection_loss‚1.9215665022532145 .loss/FBt3/detection_loss‚0.722765642173943 epoch‚5 2loss/SRqqt4/assignment_loss‚1.5421027799501368 .loss/FRt4/detection_loss‚2.259697702001123 2loss/SRqqt2/assignment_loss‚1.5267980641788907 2loss/SRqqt3/assignment_loss‚1.1458599842670891 /loss/FRt3/detection_loss‚1.7995777690472703 .loss/FBt4/detection_loss‚3.604022741317749 trainer/global_step‚6749 _step‚554 /loss/FRt4/assignment_loss‚3.748958295466853 /loss/FBt1/detection_loss‚3.4801499951969492 -loss/FBt2/detection_loss‚1.95361508263482 $ _timestamp‚1.788450232447417e+09 1loss/FBt1/assignment_loss‚0.27593692822835114 1loss/SRqqt1/detection_loss‚2.3626336509531196 &loss/total_loss‚46.469668417377136 /loss/FRt1/assignment_loss‚4.089576334953308 0loss/FRt3/assignment_loss‚1.5250166269640129 1loss/SRqqt1/assignment_loss‚1.921026239792506 1loss/SRqqt3/detection_loss‚1.0024175678217222 _runtime‚5347.918920076û‡lj\½Ö /loss/FRt3/detection_loss‚1.7995777690472703 .loss/FBt4/detection_loss‚3.604022741317749 trainer/global_step‚6749 _step‚554 /loss/FRt4/assignment_loss‚3.748958295466853 /loss/FBt1/detection_loss‚3.4801499951969492 -loss/FBt2/detection_loss‚1.95361508263482 $ _timestamp‚1.788450232447417e+09 1loss/FBt1/assignment_loss‚0.27593692822835114 1loss/SRqqt1/detection_loss‚2.3626336509531196 &loss/total_loss‚46.469668417377136 /loss/FRt1/assignment_loss‚4.089576334953308 0loss/FRt3/assignment_loss‚1.5250166269640129 1loss/SRqqt1/assignment_loss‚1.921026239792506 1loss/SRqqt3/detection_loss‚1.0024175678217222 _runtime‚5347.918920076 /loss/FRt2/assignment_loss‚3.427400556632451 1loss/FBt2/assignment_loss‚0.31998274330410936 0loss/FBt4/assignment_loss‚0.9404360577462967 0loss/SRqqt4/detection_loss‚1.952470481881629 0loss/FBt3/assignment_loss‚0.2137387490003521 /loss/FRt1/detection_loss‚2.9986498022079466 /loss/FRt2/detection_loss‚1.7352681194032942 1loss/SRqqt2/detection_loss‚1.9215665022532145 .loss/FBt3/detection_loss‚0.722765642173943 epoch‚5 2loss/SRqqt4/assignment_loss‚1.5421027799501368 .loss/FRt4/detection_loss‚2.259697702001123 2loss/SRqqt2/assignment_loss‚1.5267980641788907 2loss/SRqqt3/assignment_loss‚1.1458599842670891ªÖÈò¾‚(:… ¿«æÔ¡Üý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.95' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.79 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÏZ•ëy¿‚(:î ¿«æÔË¥Âÿ disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚21460.47412109375 disk.rbd1.in‚0( proc.memory.availableMB‚ 957.05078125% disk./.usageGB‚301.77437591552734 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.sda.out‚ 206.51953125 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0$ memory_percent‚94.15862560272217" proc.memory.rssMB‚ 2621.6484375 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.dm-1.out‚ 206.51953125 disk.drbd1716.in‚0* proc.memory.percent‚16.001272201538086 cpu‚25.332380023250472 disk.nvme0c0n1.in‚0.078125( disk.nvme0c0n1.out‚21460.47412109375 disk.sda3.out‚ 206.51953125 disk.rbd0.out‚0 disk.dm-1.in‚ 0.01171875 network.sent‚ 1.5996963e+07 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.rbd1.out‚ 29.0166015625 disk.rbd0.in‚0 disk.sda1.in‚0 network.recv‚ 1.935585e+06* disk./.usagePercent‚20.252331573257642 disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875& disk.nvme0n1.out‚21460.474121093757D&“À‚(:ˆ ΫæÔøÇúü gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.661' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.7322 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0W|Ñ-vÁ‚(:ë ΫæÔ“ʶ€* disk./.usagePercent‚20.253658716315144 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.rbd1.in‚0# disk.dm-0.out‚21514.55224609375 disk.sda.out‚ 207.12109375 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875 cpu‚25.33158684896227 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 29.0166015625% disk./.usageGB‚301.79415130615234( disk.nvme0c0n1.out‚21514.55224609375 disk.sda3.out‚ 207.12109375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 network.sent‚ 1.6106268e+07 disk.drbd2080.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚21514.55224609375$ memory_percent‚94.16017532348633 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 207.12109375& proc.memory.availableMB‚ 956.796875* proc.memory.percent‚16.001272201538086 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0" proc.memory.rssMB‚ 2621.6484375 proc.cpu.threads‚17 network.recv‚ 1.943173e+065eÍi2Â- _wandbruntime‚5383  _runtime‚5383CürÐÇÃÁ & lr-AdamW/pg1‚6.060956109200564e-05 & lr-AdamW/pg2‚6.060956109200564e-05 trainer/global_step‚6799 % _timestamp‚1.7884502684321828e+09 _runtime‚5383.903794659 _step‚555?6°fÌÄÆ trainer/global_step‚6799 % _timestamp‚1.7884502684321828e+09 _runtime‚5383.903794659 _step‚555 & lr-AdamW/pg1‚6.060956109200564e-05 & lr-AdamW/pg2‚6.060956109200564e-05«Τ§s2Å- _wandbruntime‚5383  _runtime‚5383 Ksåï%ÆÛO =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _step‚556 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚32466698 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚437144591176192 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚98720286 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚400508610613248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚98720288 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚394819017721344 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚394819035724288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚437144772448768 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚400507936247808 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚98720286 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 128267339 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5968244968448 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5689574888960 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚98720288 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 _runtime‚5383.904751805 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5688918526464 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚390149282003382 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5689574888960 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚400508610613248 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚384481305435136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5668631006002 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚431176503540224 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 128247301 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 128267339 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚394819017721344 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚400507936247808 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5688918526464 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 128247301 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚57596860 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5667976568246 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚431176527480320 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 226987627 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚394819035724288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5968087635968 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚90063560 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚57596862 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚32466256 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚390149953480498 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚384481322474496 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚90063116 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 GADeviceStatsMonitor.on_train_batch_start/overs[ó'òize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 trainer/global_step‚6799 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444™Gbæ'ÇàO eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚431176527480320 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 226987627 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚394819035724288 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5968087635968 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚90063560 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚57596862 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚32466256 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚390149953480498 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚384481322474496 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚90063116 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 trainer/global_step‚6799 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _step‚556 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚32466698 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚437144591176192 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚98720286 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚400508610613248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚98720288 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚394819017721344 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚394819035724288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚437144772448768 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚400507936247808 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚98720286 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 128267339 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5968244968448 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5689574888960 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚98720288 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 _runtime‚5383.904751805 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5688918526464 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚390149282003382 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5689574888960 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚400508610613248 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚384481305435136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5668631006002 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚431176503540224 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 128247301 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 128267339 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚394819017721344 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚400507936247808 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5688918526464 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 128247301 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚57596860 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5667976568246 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944¬|cËr2È- _wandbruntime‚5384  _runtime‚5384cI ê&ÉäM XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚437207104223744 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5969111459328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚98734304 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚435 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚390205674046230 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 _runtime‚ 5384.59352457 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚390205002568906 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚431238178217984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚90075835 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 128265755 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚384536226605056 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚98734302 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚394875370760192 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5689737954304 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚384536209565696 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5689737954304 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 227020123 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5690394330112 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32471242 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5669447441174 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚57605028 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚394875388763136 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 128285819 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚437207289677312 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚394875370760192 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚32470807 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 227000057 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚98734302 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5668793003210 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 128265755 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 128285819 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚394875388763136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 227000057 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚400565783093248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚400565108714496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 % _timestamp‚1.7884502691215641e+09 ï&ÊéM XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 128265755 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 128285819 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚394875388763136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 227000057 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚400565783093248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚400565108714496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 % _timestamp‚1.7884502691215641e+09 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚437207104223744 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5969111459328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚98734304 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚435 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚390205674046230 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 _runtime‚ 5384.59352457 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚390205002568906 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚431238178217984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚90075835 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 128265755 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚384536226605056 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚98734302 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚394875370760192 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5689737954304 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚384536209565696 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5689737954304 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 227020123 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5690394330112 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32471242 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5669447441174 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚57605028 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚394875388763136 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 128285819 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚437207289677312 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚394875370760192 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚32470807 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 227000057 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚98734302 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5668793003210­R82Ë- _wandbruntime‚5384  _runtime‚5384®Ù¹\ÌÖ 1loss/FBt3/assignment_loss‚0.24728625522054862 epoch‚5 trainer/global_step‚6799 _step‚558 /loss/FRt3/assignment_loss‚1.967528443844592 2loss/SRqqt4/assignment_loss‚1.3849250914181692 /loss/FRt1/detection_loss‚3.0148372888565063 /loss/FBt1/detection_loss‚2.4774814283146576 /loss/FRt1/assignment_loss‚4.282658777236938 /loss/FBt1/assignment_loss‚0.577047435207439 /loss/FRt3/detection_loss‚1.6570097204969554 1loss/SRqqt4/detection_loss‚1.5068832042358689 /loss/FBt2/detection_loss‚2.1585960260459354 /loss/FBt3/detection_loss‚0.7367037432268262 %loss/total_loss‚45.65104138851716 0loss/FRt4/assignment_loss‚3.2487161317650153 2loss/SRqqt3/assignment_loss‚1.1333902880981348 0loss/FBt4/assignment_loss‚1.2013318265477722 /loss/FRt2/detection_loss‚2.1852879896759987 /loss/FBt4/detection_loss‚2.3989094893137612 /loss/FRt4/detection_loss‚2.1949984340035185 1loss/SRqqt1/detection_loss‚2.4833223634474986 0loss/SRqqt2/detection_loss‚2.027599736370824 1loss/SRqqt3/detection_loss‚1.0776515650036542 % _timestamp‚1.7884502691231656e+09 _runtime‚5384.594683858 2loss/SRqqt1/assignment_loss‚2.1166959288152487 0loss/FBt2/assignment_loss‚0.6063357788537108 0loss/FRt2/assignment_loss‚3.4027697064659814 2loss/SRqqt2/assignment_loss‚1.5630747360516002°Ä°7WÍÛ /loss/FRt3/assignment_loss‚1.967528443844592 2loss/SRqqt4/assignment_loss‚1.3849250914181692 /loss/FRt1/detection_loss‚3.0148372888565063 /loss/FBt1/detection_loss‚2.4774814283146576 /loss/FRt1/assignment_loss‚4.282658777236938 /loss/FBt1/assignment_loss‚0.577047435207439 /loss/FRt3/detection_loss‚1.6570097204969554 1loss/SRqqt4/detection_loss‚1.5068832042358689 /loss/FBt2/detection_loss‚2.1585960260459354 /loss/FBt3/detection_loss‚0.7367037432268262 %loss/total_loss‚45.65104138851716 0loss/FRt4/assignment_loss‚3.2487161317650153 2loss/<ë½ SRqqt3/assignment_loss‚1.1333902880981348 0loss/FBt4/assignment_loss‚1.2013318265477722 /loss/FRt2/detection_loss‚2.1852879896759987 /loss/FBt4/detection_loss‚2.3989094893137612 /loss/FRt4/detection_loss‚2.1949984340035185 1loss/SRqqt1/detection_loss‚2.4833223634474986 0loss/SRqqt2/detection_loss‚2.027599736370824 1loss/SRqqt3/detection_loss‚1.0776515650036542 % _timestamp‚1.7884502691231656e+09 _runtime‚5384.594683858 2loss/SRqqt1/assignment_loss‚2.1166959288152487 0loss/FBt2/assignment_loss‚0.6063357788537108 0loss/FRt2/assignment_loss‚3.4027697064659814 2loss/SRqqt2/assignment_loss‚1.5630747360516002 1loss/FBt3/assignment_loss‚0.24728625522054862 epoch‚5 trainer/global_step‚6799 _step‚558®Ouš“΂(:ˆ Ý«æÔ‹žÿý gpu.0.gpu‚37.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚174.627' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.9254 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0MŠ”lÏ‚(:á Ý«æÔ䞘ÿ disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda.out‚ 207.78125 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0 disk.rbd1.out‚ 29.0791015625 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 207.78125 disk.rbd0.out‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0# disk.dm-0.out‚21573.62451171875 disk.sda1.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0) disk./.usagePercent‚20.25483891104799 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda2.out‚0# memory_percent‚94.1638708114624" proc.memory.rssMB‚ 2621.6484375 network.sent‚ 1.6111412e+07 network.recv‚ 1.94664e+06$ disk./.usageGB‚301.8117370605469& disk.nvme0n1.out‚21573.62451171875 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 207.78125 disk.drbd1716.in‚0( disk.nvme0c0n1.out‚21573.62451171875 disk.sda1.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125( proc.memory.availableMB‚ 956.19140625* proc.memory.percent‚16.001272201538086 cpu‚25.330772372906786 disk.sda.in‚ 0.01171875Ù,K“Ђ(:ˆ ì«æÔêúêü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.047' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8094 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 K£ pÑ‚(:å ì«æÔù¬Ùÿ network.sent‚ 1.6219558e+07* disk./.usagePercent‚20.255709336664406 disk.drbd1716.in‚0 disk.drbd1716.out‚0* proc.memory.percent‚16.001272201538086 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda2.out‚0 disk.rbd1.out‚ 29.1650390625# disk.dm-0.out‚21620.57373046875 disk.sda1.out‚0 disk.rbd1.in‚0" proc.memory.rssMB‚ 2621.6484375 cpu‚25.329991686146382 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚21620.57373046875 disk.sda.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 network.recv‚ 1.95291e+06 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 208.56640625 disk.sda3.out‚ 208.56640625 disk.drbd2266.out‚0$ memory_percent‚94.16275024414062# proc.memory.availableMB‚956.375 proc.cpu.threads‚17" disk./.usageGB‚301.82470703125 disk.drbd2265.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 208.56640625& disk.nvme0n1.out‚21620.57373046875ýªŒ™‘Ò‚(:† û«æÔàÈ÷ü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚133.13' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.626 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ûª²KyÓ‚(:î û«æÔåªÿÿ disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.nvme0c0n1.in‚0.078125# disk.dm-0.out‚21681.68017578125 disk.sda1.out‚0* proc.memory.percent‚16.001272201538086 cpu‚25.329217438902266 network.recv‚ 1.956122e+06* disk./.usagePercent‚20.256569265971923 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.sda2.out‚0( proc.memory.availableMB‚ 956.62109375( disk.nvme0c0n1.out‚21681.68017578125 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 208.87890625 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 208.87890625 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875$ memory_percent‚94.16124820709229" proc.memory.rssMB‚ 2621.6484375 disk.sda.out‚ 208.87890625& disk.nvme0n1.out‚21681.68017578125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.sda3.in‚ 0.01171875 proc.cpu.threads‚17 network.sent‚ 1.6224691e+07% disk./.usageGB‚301.83752059936523 disk.rbd1.in‚0 disk.rbd1.out‚ 29.4384765625 disk.drbd1716.out‚0 disk.drbd2266.in‚0”ƒ¯ 2Ô- _wandbruntime‚5420  _runtime‚54203G€ÉÕà _step‚559 ' lr-AdamW/pg1‚5.9696303188035386e-05 ' lr-AdamW/pg2‚5.9696303188035386e-05 trainer/global_step‚6849 % _timestamp‚1.7884503054118314e+09 _runtime‚5420.883410998qsÎÖÈ ' lr-AdamW/pg2‚5.9696303188035386e-05 trainer/global_step‚6849 % _timestamp‚1.7884503054118314e+09 _runtime‚5420.883410998 _step‚559 ' lr-AdamW/pg1‚5.9696303188035386e-05¯àô6C2×- _wandbruntime‚5420  _runtime‚5420©:×2ß'ØÙO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5730560694784 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 228614479 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 129173353 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚90700016 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6011265096704 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 # _timestamp‚1.78845030541251e+09 _step‚560 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 129173353 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 129193391 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚403367248361472 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚403366573996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚397636687666688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚99421086 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚392935323870790 _runtime‚5420.884312108 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5729904332288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚32694302 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 228594439 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚403366573996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚90699562 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚32694754 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚403367248361472 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚58005262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 129193391 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚6849 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚387226511963136 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5708811907654 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚392935995347906 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚99421088 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚397636687666688 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5709466345410 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚403367248361472 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚403366573996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚397636687666688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚99421086 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚392935323870790 _runtime‚5420.884312108 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5729904332288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚32694302 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 228594439 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚403366573996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚90699562 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚32694754 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚403367248361472 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚58005262 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 129193391 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚6849 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚387226511963136 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5708811907654 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚392935995347906 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚99421088 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚397636687666688 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5709466345410 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 228626909 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32699191 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 228646975 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚392991044436314 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚99435102 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5730723760128 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 129211871 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚397693040705536 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚387281433133056 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 $ _timestamp‚1.788450306089712e+09 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚397693040705536 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 129211871 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 129191807 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5731380135936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚58013428 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚392991715913638 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6012128110080 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 228646975 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚99435104 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5731380135936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚387281416093696 trainer/global_step‚6849 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5709628342618 _runtime‚5421.562046963 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚90712172 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0ë;®ï&ÜéM K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 129191807 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚434320691161088 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6011974985216 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚561 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚58013430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚440332843211264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚440332666146304 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚403423746462720 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 228626909 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚32699191 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 228646975 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚392991044436314 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚99435102 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5730723760128 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 129211871 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚397693040705536 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚387281433133056 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 $ _timestamp‚1.788450306089712e+09 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚397693040705536 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 129211871 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 129191807 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5731380135936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚58013428 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚392991715913638 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6012128110080 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 228646975 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚99435104 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5731380135936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚387281416093696 trainer/global_step‚6849 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5709628342618 _runtime‚5421.562046963 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚90712172 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5749647247062 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚406225211744256 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚406225886109696 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5771546500608 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6054457541632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚58413660 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 130099405 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 100121888 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5770890138112 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6054287626240 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5749647247062 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚406225211744256 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_sùWitart/allocated_bytes.all.allocated‚406225886109696 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5771546500608 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6054457541632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚58413660 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 130099405 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 100121888 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5770890138112 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6054287626240 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚395777086303722 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚406283058589696 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5751118119990 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 130117859 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚437403251984384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚32925778 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚6899 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5750463682026 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚400510674644992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚58421830 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5772365941760 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚58421828 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 230273827 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 230273827 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 5458.04860916 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 100135902 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5771709565952 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 230253761 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 100135904 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚459 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚400510674644992 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 100135904 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚437403228044288 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚565 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 130117859 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 $ _timestamp‚1.788450342576653e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 130137923 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚406283058589696 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 230253761 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚91347606 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚91347606 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚395777086303722 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚406283058589696 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5751118119990 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 130117859 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚437403251984384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚32925778 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚6899 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5750463682026 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚400510674644992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚58421830 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5772365941760 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚58421828 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 230273827 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 230273827 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 5458.04860916 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 100135902 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5771709565952 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 230253761 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 100135904 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚459 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚400510674644992 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 100135904 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚437403228044288 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚565 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 130117859 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 $ _timestamp‚1.788450342576653e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 130137923 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚406283058589696 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 230253761 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚409083849492480 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 131025457 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚58822060 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚446521628960768 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5811875943936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚392716942058496 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5790482586470 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 100822686 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5812532306432 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 100822686 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚33149495 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6097676297728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚403271991551488 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚409083849492480 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 131025457 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 100822688 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 231868183 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 131045495 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5791137024226 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5812532306432 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚6949 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 % _timestamp‚1.7884503785544438e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚403271991551488 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚446521814427648 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚409083849492480 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 131025457 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚58822060 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚446521628960768 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5811875943936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚392716942058496 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5790482586470 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 100822686 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5812532306432 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 100822686 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚33149495 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6097676297728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚403271991551488 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚409083849492480 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 131025457 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 100822688 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 231868183 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 131045495 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5791137024226 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5812532306432 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚6949 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 % _timestamp‚1.7884503785544438e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚403271991551488 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚446521814427648 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚91984270 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚409141696337920 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5813351747584 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚33154042 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚409141696337920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚91984727 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 131063975 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884503792476063e+09 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚403328326587392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 100836702 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 trainer/global_step‚6949 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 231900679 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5791299021434 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5812695371776 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 231880613 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 100836704 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚440485788867584 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚5494.719624958 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5812695371776 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚446584322072576 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6098363302912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 131063975 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segment®";Ñàs.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0a*Fô&þîM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚409141021959168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚33154497 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6098533204992 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚403328344590336 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚58830228 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚446584128230400 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5791953459398 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚403328326587392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 100836702 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 trainer/global_step‚6949 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 231900679 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5791299021434 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5812695371776 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 231880613 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 100836704 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚440485788867584 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚5494.719624958 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5812695371776 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚446584322072576 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6098363302912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 131063975 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚440485764927488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚91984270 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚409141696337920 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5813351747584 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚33154042 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚409141696337920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚91984727 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 131063975 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884503792476063e+09 PDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 101523488 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚33377282 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 trainer/global_step‚6999 _runtime‚5530.481481248 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5852861749760 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚411943161606144 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5853518112256 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚411942487240704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚401294120950130 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚449647679341056 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚406089643493888 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚411943161606144 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5853518112256 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 101523486 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6140842801152 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 233474995 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚92607742 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚33377739 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 131971547 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6141004327936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚395462131547136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5831972363634 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 131951509 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚406089625490944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚59230460 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5831317925878 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496†ƒ›ä‹ãO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5853518112256 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚411942487240704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚401294120950130 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚449647679341056 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚406089643493888 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚411943161606144 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5853518112256 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 101523486 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6140842801152 WJDeviûŸ¦ceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 233474995 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚92607742 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚33377739 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 131971547 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6141004327936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚395462131547136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5831972363634 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 131951509 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚406089625490944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚59230460 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5831317925878 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚406089643493888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 233495035 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚401293449473014 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 101523488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 $ _timestamp‚1.788450415009583e+09 _step‚572 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 101523488 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚33377282 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 trainer/global_step‚6999 _runtime‚5530.481481248 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5852861749760 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚411943161606144 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128¼äa€2Œ- _wandbruntime‚5531  _runtime‚5531ÃÞäí&çM REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚395517052717056 _step‚573 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚92620511 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚449710196056576 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚401349841515862 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚59238628 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 131969963 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚401349170038538 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5854337553408 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚33381883 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚406145996532736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5832788798806 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 131990027 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 233527531 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚395517035677696 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5853681177600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 101537502 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚92620974 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚443568325750784 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚463 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚59238630 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚33382344 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 233527531 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚406145978529792 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5853681177600 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 101537504 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 233507465 trainer/global_step‚6999 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚411999659707392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 233507465 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚406145996532736 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 101537504 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 5531.17379757 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚412000334086144 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5854337553408 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚443568301810688 û dò&ŽìM E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚401349841515862 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚59238628 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 131969963 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚401349170038538 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5854337553408 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚33381883 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚406145996532736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5832788798806 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 131990027 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 233527531 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚395517035677696 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5853681177600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 101537502 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚92620974 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚443568325750784 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚463 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚59238630 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚33382344 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 233527531 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚406145978529792 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5853681177600 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 101537504 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 233507465 trainer/global_step‚6999 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚411999659707392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 233507465 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚406145996532736 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 101537504 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 5531.17379757 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚412000334086144 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5854337553408 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚443568301810688 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚395517052717056 _step‚573 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚92620511 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚449710196056576 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964½eøÊ2- _wandbruntime‚5531  _runtime‚5531½ äUÏ 2loss/SRqqt2/assignment_loss‚1.8264048394229677 1loss/FBt4/assignment_loss‚0.26346245408058167 2loss/SRqqt4/assignment_loss‚1.2915697392475392 .loss/FBt2/detection_loss‚1.321688582499822 trainer/global_step‚6999 % _timestamp‚1.7884504157033234e+09 0loss/SRqqt1/assignment_loss‚2.54717430486702 1loss/SRqqt4/detection_loss‚1.7907608891172069 /loss/FBt3/detection_loss‚0.5503671664372086 _step‚574 -loss/FRt2/detection_loss‚2.54800196037148 1loss/SRqqt1/detection_loss‚2.7078060782872715 1loss/SRqqt3/detection_loss‚0.9714083712495548 2loss/SRqqt3/assignment_loss‚1.0167156280556195 1loss/FBt1/assignment_loss‚0.49134117082321893 .loss/FRt3/detection_loss‚1.521714698523283 /loss/FRt4/detection_loss‚2.1993133172392847 .loss/FBt1/detection_loss‚3.415740180760622 -loss/FBt4/detection_loss‚7.44997638463974 /loss/FRt1/assignment_loss‚4.331940999031067 0loss/FRt4/assignment_loss‚2.9108376693725586 0loss/SRqqt2/detection_loss‚2.091709783838855 %loss/total_loss‚50.67785590648526 epoch‚5 _runtime‚5531.174855856 /loss/FRt2/assignment_loss‚4.284155943176963 0loss/FRt3/assignment_loss‚1.1501988386208157 0loss/FBt2/assignment_loss‚0.6626438359419485 1loss/FBt3/assignment_loss‚0.19812937209294687 .loss/FRt1/detection_loss‚3.134793698787689ÙÆ{ Z‘Ô 0loss/FRt3/assignment_loss‚1.1501988386208157 0loss/FBt2/assignment_loss‚0.6626438359419485 1loss/FBt3/assignment_loss‚0.19812937209294687 .loss/FRt1/detection_loss‚3.134793698787689 2loss/SRqqt2/assignment_loss‚1.8264048394229677 1loss/FBt4/assignment_loss‚0.26346245408058167 2loss/SRqqt4/assignment_loss‚1.2915697392475392 .loss/FBt2/detection_loss‚1.321688582499822 trainer/global_step‚6999 % _timestamp‚1.7884504157033234e+09 0loss/SRqqt1/assignment_loss‚2.54717430486702 1loss/SRqqt4/detection_loss‚1.7907608891172069 /loss/FBt3/detection_loss‚0.5503671664372086 _step‚574 -loss/FRt2/detection_loss‚2.54800196037148 1loss/SRqqt1/detection_loss‚2.7078060782872715 1loss/SRqqt3/detection_loss‚0.9714083712495548 2loss/SRqqt3/assignment_loss‚1.0167156280556195 1loss/FBt1/assignment_loss‚0.49134117082321893 .loss/FRt3/detection_loss‚1.521714698523283 /loss/FRt4/detection_loss‚2.1993133172392847 .loss/FBt1/detection_loss‚3.415740180760622 -loss/FBt4/detection_loss‚7.44997638463974 /loss/FRt1/assignment_loss‚4.331940999031067 0loss/FRt4/assignment_loss‚2.9108376693725586 0loss/SRqqt2/detection_loss‚2.091709783838855 %loss/total_loss‚50.67785590648526 epoch‚5 _runtime‚5531.174855856 /loss/FRt2/assignment_loss‚4.284155943176963¾\2²ÿž’‚(:“ ó¬æÔ°ÈÇþ gpu.0.gpu‚48.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚194.336' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚38.867200000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ò}«1l“‚(:á ó¬æÔÛŽúþ$ disk./.usageGB‚301.9290351867676 disk.sda1.in‚0 disk.drbd2080.out‚0" proc.memory.rssMB‚ 2621.6484375 network.sent‚ 1.6579569e+07 disk.nvme0c0n1.in‚0.078125* disk./.usagePercent‚20.262710886718345 disk.rbd0.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.drbd2266.in‚0# memory_percent‚94.1608190536499* proc.memory.percent‚16.001272201538086 proc.cpu.threads‚17 network.recv‚ 1.995761e+06 disk.rbd0.in‚0" disk.dm-0.out‚22097.9970703125' disk.nvme0c0n1.out‚22097.9970703125 disk.sda3.in‚ 0.01171875( proc.memory.availableMB‚ 956.69140625 disk.sda2.out‚0 disk.dm-0.in‚0.078125 disk.dm-1.out‚ 213.578125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚22097.9970703125 disk.drbd2265.in‚0 disk.drbd2266.out‚0 disk.sda.out‚ 213.578125 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd2265.out‚0 disk.sda3.out‚ 213.578125 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.rbd1.out‚ 30.166015625 cpu‚25.32311566842714n´µž”‚(:“ ‚­æÔÉœôü gpu.0.gpu‚35.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚186.548' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.309599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0x.K{r•‚(:ç ‚­æÔáªàþ disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.sda1.in‚0 disk.sda2.in‚0* proc.memory.percent‚16.001272201538086* disk./.usagePercent‚20.263373946232026 disk.dm-1.out‚ 214.2109375 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.sda2.out‚0" proc.memory.rssMB‚ 2621.6484375 network.recv‚ 2.003767e+06' disk.nvme0c0n1.out‚22148.2275390625% disk.nvme0n1.out‚22148.2275390625 disk.drbd2080.in‚0 cpu‚25.322383264870336 disk.rbd1.in‚0 disk.rbd0.in‚0" disk.dm-0.out‚22148.2275390625$ memory_percent‚94.16539669036865 network.sent‚ 1.6689145e+07 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125( proc.memory.availableMB‚ 955.94140625% disk./.usageGB‚301.93891525268555 disk.rbd1.out‚ 30.166015625 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.sda3.out‚ 214.2109375 disk.sda.out‚ 214.2109375 proc.cpu.threads‚17‹6D…ž–‚(:“ ‘­æÔçãý gpu.0.gpu‚42.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚131.907' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.381400000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÄƤl—‚(:á ‘­æÔƒá¢ÿ* disk./.usagePercent‚20.264001676705984 disk.rbd0.in‚0 disk.sda.out‚ 214.546875 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 network.recv‚ 2.00722e+06 disk.rbd0.out‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚22228.8701171875 disk.rbd1.in‚0 disk.dm-1.out‚ 214.546875 disk.nvme0c0n1.in‚0.078125% disk./.usageGB‚301.94826889038086 disk.sda1.in‚0 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚22228.8701171875 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 30.166015625 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875# memory_percent‚94.1608190536499* proc.memory.percent‚16.001272201538086% disk.nvme0n1.out‚22228.8701171875 disk.drbd1716.in‚0 disk.drbd2080.in‚0( proc.memory.availableMB‚ 956.69140625 cpu‚25.32164800526058 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 214.546875 disk.drbd2266.in‚0 disk.drbd2266.out‚0" proc.memory.rssMB‚ 2621.6484375 proc.cpu.threads‚17 disk.drbd2265.in‚0 network.sent‚ 1.6694357e+07–39Õ2˜- _wandbruntime‚5567  _runtime‚55672Í–NÇ™Á _step‚575 & lr-AdamW/pg1‚5.605277151601517e-05 & lr-AdamW/pg2‚5.605277151601517e-05 trainer/global_step‚7049 % _timestamp‚1.7884504518956552e+09 _runtime‚5567.367238062rb#I|šÆ trainer/global_step‚7049 % _timestamp‚1.7884504518956552e+09 _runtime‚5567.367238062 _step‚575 & lDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚59638862 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚446589187956224 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚408907277433344 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚404080162817538 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5893847555584 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5872807703042 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 102224286 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 132877561 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚33604164 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 235101847 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5894503918080 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 102224288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚437 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚408907295436288 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚33604599 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚452773227724288 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚446589211896320 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚404079491340422 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚414801799354368 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚414801799354368 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚414801124988928 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0½E:‡ê'äO SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 132897599 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 132897599 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚59638860 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6184182420480 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _runtime‚5567.368149512 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 102224286 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 166592512 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 142652416 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 102224288 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚408907295436288 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5893847555584 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6184039768064 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚398207355114496 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚398207338075136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5894503918080 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 235121887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚59638862 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚446589187956224 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚408907277433344 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚404080162817538 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5893847555584 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 5872807703042 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 102224286 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 132877561 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚33604164 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 235101847 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5894503918080 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 102224288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚437 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚408907295436288 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚33604599 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚452773227724288 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚446589211896320 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚404079491340422 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚414801799354368 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚414801799354368 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚414801124988928Àΰ+¬2ž- _wandbruntime‚5568  _runtime‚5568mQ~ƒï&ŸéM SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 132916079 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5872969700250 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 132896015 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚404135211905946 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 235154383 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 102238302 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5895323359232 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 132896015 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚59647028 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _step‚577 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5873624138214 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 102238304 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6185055341056 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚93255624 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚398262242205696 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5894666983424 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 132916079 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚452835917975040 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚446650838693888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 102238304 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚408963648475136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 102238302 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚446650862633984 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7049 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5895323359232 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚398262259245056 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5894666983424 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 235134317 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚414858971834368 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 235154383 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6184887536128 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚408963630472192 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚414858297455616 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚93256071 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚33608596 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚59647030 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚404135883383270 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 % _timestamp‚1.7884504526012995e+09 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚408963630472192 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚452835726230016 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096Ù¡] îM ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚93255624 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚398262242205696 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5894666983424 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 132916079 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚452835917975040 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚446650838693888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 102238304 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚408963648475136 TFDeviceStatsMonito üs—r.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 102238302 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚446650862633984 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚7049 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5895323359232 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚398262259245056 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5894666983424 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 235134317 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚414858971834368 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 235154383 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6184887536128 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚408963630472192 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚414858297455616 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚93256071 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚33608596 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚59647030 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚404135883383270 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 % _timestamp‚1.7884504526012995e+09 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚408963630472192 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚452835726230016 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 132916079 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5872969700250 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 132896015 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚404135211905946 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 235154383 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 102238302 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5895323359232 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 132896015 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚59647028 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _step‚577 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5873624138214 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 102238304 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6185055341056 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚417659762737152 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 236748739 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚93879598 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚411724929375744 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚449671748779520 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚60047260 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 % _timestamp‚1.7884504888480594e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 102925088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6227264073216 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚406865533207830 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚449671724839424 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚455899012852736 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 133803613 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚33832338 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚400952561642496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚93880039 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 102925088 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚411724947378688 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚455898825288704 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 102925086 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚417660437102592 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 133803613 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 236728699 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5935489723904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5935489723904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚60047262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚411724929375744 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 102925086 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6227100449280 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 133823651 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5934833361408 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088Ö¦µ= ­äO MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚449671724839424 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚455899012852736 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 133803613 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚33832338 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚400952561642496 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚93880039 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 102925088 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚411724947378688 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚455898825288704 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 102925086 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚417660437102592 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 133803613 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 236728699 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5935489723904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5935489723904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚60047262 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚411724929375744 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 102925086 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6227100449280 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 133823651 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5934833361408 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 59O“úó­12988604694 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚417659762737152 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 236748739 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚93879598 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚411724929375744 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚449671748779520 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚60047260 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 % _timestamp‚1.7884504888480594e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 102925088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6227264073216 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚406865533207830ÄA#2®- _wandbruntime‚5605  _runtime‚5605³¬ýï&¯éM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5935652789248 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚406921925250678 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 102939104 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5935652789248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚455961526165504 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚33837387 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6227967232000 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚411781300417536 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚93892817 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 133822067 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚417716935203840 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5936309165056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚449733399517184 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚93892367 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 trainer/global_step‚7099 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 236761169 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6228126648320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚33836939 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 133842131 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚401007448733696 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 236781235 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 236781235 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5936309165056 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 133822067 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _step‚581 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚411781300417536 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚401007465773056 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 102939102 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 102939102 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚455961342809088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414IÁzô&°îM RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 trainer/global_step‚7099 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 236761169 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6228126648320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚33836939 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 133842131 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚401007448733696 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 236781235 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 236781235 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5936309165056 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 133822067 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _step‚581 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚411781300417536 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚401007465773056 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 102939102 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 102939102 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚455961342809088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 5935652789248 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚406921925250678 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 102939104 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 5935652789248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚455961526165504 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚33837387 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6227967232000 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚411781300417536 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚93892817 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 133822067 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚417716935203840 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5936309165056 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚449733399517184 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚93892367ÅÙi®2±- _wandbruntime‚5605  _runtime‚5605Zöÿ=W²Ñ 0loss/FRt1/assignment_loss‚3.9310155630111696 /loss/FRt4/assignment_loss‚3.358852345319021 2loss/SRqqt2/assignment_loss‚1.2002197502310885 /loss/FRt3/detection_loss‚1.8170286444376926 _step‚582 /loss/FRt2/assignment_loss‚2.710616463422775 0loss/FBt1/assignment_loss‚0.6516820232959617 0loss/FBt2/assignment_loss‚0.6792972030738856 /loss/FRt4/detection_loss‚2.4497143789416267 /loss/FBt4/detection_loss‚2.6821662545204163 trainer/global_step‚7099 _runtime‚5605.015327459 2loss/SRqqt1/assignment_loss‚1.9323473379418656 1loss/FBt3/assignment_loss‚0.26509432874784106 1loss/SRqqt4/detection_loss‚1.7890620999362157 1loss/SRqqt3/detection_loss‚1.3235668296378755 .loss/FBt1/detection_loss‚2.505421045151624 epoch‚5 /loss/FRt1/detection_loss‚3.1418348968029024 .loss/FBt3/detection_loss‚0.713793982292215 $loss/total_loss‚43.9011281475409 /loss/FRt2/detection_loss‚2.0884957522153855 0loss/SRqqt1/detection_loss‚2.653524841811206 2loss/SRqqt3/assignment_loss‚1.0888696645223905 0loss/FBt4/assignment_loss‚0.2070711581094656 .loss/FBt2/detection_loss‚1.435972880323728 0loss/FRt3/assignment_loss‚1.7576540355779686 1loss/SRqqt4/assignment_loss‚1.355982160317185 1loss/SRqqt2/detection_loss‚2.1618445078993953 $ _timestamp‚1.788450489543855e+09ºüD\³Ö 1loss/SRqqt4/assignment_loss‚1.355982160317185 1loss/SRqqt2/detection_loss‚2.1618445078993953 $ _timestamp‚1.788450489543855e+09 0loss/FRt1/assignment_loss‚3.9310155630111696 /loss/FRt4/assignment_loss‚3.358852345319021 2loss/SRqqt2/assignment_loss‚1.2002197502310885 /loss/FRt3/detection_loss‚1.8170286444376926 _step‚582 /loss/FRt2/assignment_loss‚2.710616463422775 0loss/FBt1/assignment_loss‚0.6516820232959617 0loss/FBt2/assignment_loss‚0.6792972030738856 /loss/FRt4/detection_loss‚2.4497143789416267 /loss/FBt4/detection_loss‚2.6821662545204163 trainer/global_step‚7099 _runtime‚5605.015327459 2loss/SRqqt1/assignment_loss‚1.9323473379418656 1loss/FBt3/assignment_loss‚0.26509432874784106 1loss/SRqqt4/detection_loss‚1.7890620999362157 1loss/SRqqt3/detection_loss‚1.3235668296378755 .loss/FBt1/detection_loss‚2.505421045151624 epoch‚5 /loss/FRt1/detection_loss‚3.1418348968029024 .loss/FBt3/detection_loss‚0.713793982292215 $loss/total_loss‚43.9011281475409 /loss/FRt2/detection_loss‚2.0884957522153855 0loss/SRqqt1/detection_loss‚2.653524841811206 2loss/SRqqt3/assignment_loss‚1.0888696645223905 0loss/FBt4/assignment_loss‚0.2070711581094656 .loss/FBt2/detection_loss‚1.435972880323728 0loss/FRt3/assignment_loss‚1.7576540355779686Æ „X“´‚(:ˆ ¾­æÔâú¸ý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.551' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.1102 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Üfxµ‚(:í ¾­æÔƒÛ÷ÿ# disk.dm-0.out‚22397.04052734375 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 30.4794921875 disk.drbd1716.out‚0 disk.rbd0.in‚0) disk./.usagePercent‚20.26597984692306 disk.drbd2266.out‚0 disk.drbd2265.out‚0$ memory_percent‚94.16143894195557( disk.nvme0c0n1.out‚22397.04052734375 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 network.sent‚ 1.6815248e+07 disk.sda3.out‚ 216.36328125 disk.drbd2265.in‚0* proc.memory.percent‚16.001272201538086 disk.sda1.in‚0( proc.memory.availableMB‚ 956.58984375" proc.memory.rssMB‚ 2621.6484375& disk.nvme0n1.out‚22397.04052734375 disk.rbd0.out‚0% disk./.usageGB‚301.97774505615234 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 network.recv‚ 2.022902e+06 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 216.36328125 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.dm-1.out‚ 216.36328125 cpu‚25.319474140418123s\_‘¶‚(:† Í­æÔ»˜ ý gpu.0.gpu‚39.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.26' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.252 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Îùc·‚(:Ø Í­æÔÃê¥ÿ disk.sda3.in‚ 0.01171875 disk.dm-0.out‚ 22449.40625* disk./.usagePercent‚20.266988516607967" disk.nvme0c0n1.out‚ 22449.40625 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.out‚0" proc.memory.rssMB‚ 2621.6484375 disk.drbd2266.in‚0 disk.sda.out‚ 216.8203125 disk.sda1.in‚0 disk.sda2.in‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125$ disk./.usageGB‚301.9927749633789 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 216.8203125 proc.cpu.threads‚17 disk.rbd1.out‚ 30.4794921875 disk.sda3.out‚ 216.8203125 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.nvme0n1.out‚ 22449.40625$ memory_percent‚94.16296482086182 network.sent‚ 1.6922988e+07 disk.rbd0.out‚0( proc.memory.availableMB‚ 956.33984375* proc.memory.percent‚16.001272201538086 cpu‚25.318676808877985 network.recv‚ 2.027968e+06 disk.drbd1716.in‚0 disk.drbd2265.in‚0M¡Gž¸‚(:“ Ü­æÔª­ûü gpu.0.gpu‚49.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.905' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.580999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0nHÖh¹‚(:Ý Ü­æÔŠŽ®€ disk.dm-1.out‚ 217.72265625 disk.sda3.out‚ 217.72265625 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0* proc.memory.percent‚16.001272201538086 disk.drbd2265.in‚0 disk.sda.out‚ 217.72265625 disk.sda2.out‚0( proc.memory.availableMB‚ 956.08203125" proc.memory.rssMB‚ 2621.6484375 network.sent‚ 1.6929461e+07 disk.rbd1.in‚0" disk.nvme0n1.out‚ 22492.9140625$ disk.nvme0c0n1.out‚ 22492.9140625 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 30.51171875 disk.drbd2080.out‚0$ disk./.usageGB‚301.9784736633301 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.dm-0.out‚ 22492.9140625$ memory_percent‚94.16453838348389 disk.drbd1716.out‚0( disk./.usagePercent‚20.2660287443621 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚0.078125 cpu‚25.318000303180387 network.recv‚ 2.032701e+06 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 proc.cpu.threads‚17œ–h2º- _wandbruntime‚5640  _runtime‚5640XKÆ»À & lr-AdamW/pg1‚5.423936233241143e-05 & lr-AdamW/pg2‚5.423936233241143e-05 trainer/global_step‚7149 % _timestamp‚1.7884505254172335e+09 _runtime‚ 5640.88884487 _step‚583}¹½®Ë¼Å % _timestamp‚1.7884505254172335e+09 _runtime‚ 5640.88884487 _step‚583 & lr-AdamW/pg1‚5.423936233241143e-05 & lr-AdamW/pg2‚5.423936233241143e-05 trainer/global_step‚7149ÇgWpr2½- _wandbruntime‚5640  _runtime‚5640‚yá ¾ÞO WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5975819167232 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚414542599321088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚452754285662720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚459024612529664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚403697768170496 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚94516819 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚94516376 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5953823944102 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚414542599321088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 103625888 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5976475529728 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5975819167232 _step‚584 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚420518400485376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚441 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚34060716 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚409652246552354 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5976475529728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚414542581318144 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 134749703 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚420518400485376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 _runtime‚5640.889785545 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚403697751131136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0v2(´é'¿ãO XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 5976475529728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚414542581318144 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 134749703 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚420518400485376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 _runtime‚5640.889785545 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚403697751131136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 5975819167232 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚414542599321088 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚452754285662720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚459024612529664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚403697768170496 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚94516819 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚94516376 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5953823944102 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚414542599321088 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 103625888 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 5976475529728 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 5975819167232 _step‚584 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚420518400485376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚441 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚34060716 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚409652246552354Èıc2À- _wandbruntime‚5641  _runtime‚5641)ñóï&ÁéM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚34065273 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 103639904 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚5641.584314083 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 134768183 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚585 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 103639902 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚459087120513536 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5977294970880 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚414598952359936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚463 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚409707295640762 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 103639902 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚420575572952064 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚414598934356992 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 238408087 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6271184113152 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6271014211072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 % _timestamp‚1.7884505261121538e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 134748119 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5954640379066 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚94529101 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚34065734 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚409707967118086 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 134748119 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5955294817030 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚414598952359936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚459086926671360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5977294970880 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 238408087 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 238388021 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚420575572952064 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚414598934356992 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 238408087 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6271184113152 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚461 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6271014211072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 % _timestamp‚1.7884505261121538e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 134748119 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5954640379066 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚94529101 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚34065734 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚409707967118086 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚µz«$0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 134748119 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 5955294817030 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚414598952359936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚459086926671360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 5977294970880 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 238408087 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 238388021 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚585 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 103639902 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚459087120513536 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 5977294970880 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚414598952359936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚463 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0É°Æ)2Ã- _wandbruntime‚5641  _runtime‚5641Ç^0ý\ÄÖ _step‚586 .loss/FBt1/detection_loss‚2.701667046546936 /loss/FBt2/detection_loss‚1.9152379264434178 &loss/total_loss‚45.650742558920896 2loss/SRqqt2/assignment_loss‚1.7108811316829102 1loss/SRqqt4/assignment_loss‚1.453424330088698 1loss/FBt1/assignment_loss‚0.29069230453947814 1loss/FBt4/assignment_loss‚0.12996758203928493 .loss/FRt1/detection_loss‚3.081007224542123 0loss/SRqqt4/detection_loss‚1.528301635881265 2loss/SRqqt1/assignment_loss‚2.3553252267837523 0loss/FBt2/assignment_loss‚0.5617578256874973 .loss/FRt4/detection_loss‚2.210910765188081 /loss/FRt1/assignment_loss‚4.341093641740304 0loss/FRt2/assignment_loss‚3.3440750869257108 /loss/FRt3/assignment_loss‚1.996386569959146 2loss/SRqqt3/assignment_loss‚1.1234860683736556 1loss/SRqqt3/detection_loss‚1.0657871502506382 /loss/FRt2/detection_loss‚2.2201049205447947 /loss/FRt3/detection_loss‚1.6428843791838046 1loss/SRqqt2/detection_loss‚1.7131188488474078 1loss/FBt3/assignment_loss‚0.23882190085235272 1loss/SRqqt1/detection_loss‚2.7365263366699217 /loss/FBt4/detection_loss‚3.0081725835800173 trainer/global_step‚7149 0loss/FRt4/assignment_loss‚3.5472399993818633 /loss/FBt3/detection_loss‚0.7338720731878359 epoch‚5 % _timestamp‚1.7884505261137724e+09 _runtime‚5641.585278242¡k£aÅÛ /loss/FRt1/assignment_loss‚4.341093641740304 0loss/FRt2/assignment_loss‚3.3440750869257108 /loss/FRt3/assignment_loss‚1.996386569959146 2loss/SRqqt3/assignment_loss‚1.1234860683736556 1loss/SRqqt3/detection_loss‚1.0657871502506382 /loss/FRt2/detection_loss‚2.2201049205447947 /loss/FRt3/detection_loss‚1.6428843791838046 1loss/SRqqt2/detection_loss‚1.7131188488474078 1loss/FBt3/assignment_loss‚0.23882190085235272 1loss/SRqqt1/detection_loss‚2.7365263366699217 /loss/FBt4/detection_loss‚3.0081725835800173 trainer/global_step‚7149 0loss/FRt4/assignment_loss‚3.5472399993818633 /loss/FBt3/detection_loss‚0.7338720731878359 epoch‚5 % _timestamp‚1.7884505261137724e+09 _runtime‚5641.585278242 _step‚586 .loss/FBt1/detection_loss‚2.701667046546936 /loss/FBt2/detection_loss‚1.9152379264434178 &loss/total_loss‚45.650742558920896 2loss/SRqqt2/assignment_loss‚1.7108811316829102 1loss/SRqqt4/assignment_loss‚1.453424330088698 1loss/FBt1/assignment_loss‚0.29069230453947814 1loss/FBt4/assignment_loss‚0.12996758203928493 .loss/FRt1/detection_loss‚3.081007224542123 0loss/SRqqt4/detection_loss‚1.528301635881265 2loss/SRqqt1/assignment_loss‚2.3553252267837523 0loss/FBt2/assignment_loss‚0.5617578256874973 .loss/FRt4/detection_loss‚2.210910765188081ÊVE“Æ‚(:ˆ ë­æÔÙÝ·ý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.482' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8964 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0)hF¯uÇ‚(:ê ë­æÔŒÚð€( disk.nvme0c0n1.out‚22551.54345703125 disk.sda3.out‚ 218.16796875 disk.drbd2266.in‚0# disk.dm-0.out‚22551.54345703125 disk.sda.out‚ 218.16796875 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.dm-1.in‚ 0.01171875* proc.memory.percent‚16.001272201538086 proc.cpu.threads‚17 network.sent‚ 1.6934689e+07* disk./.usagePercent‚20.267412465084668 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.dm-1.out‚ 218.16796875 disk.drbd2080.out‚0 disk.rbd0.in‚0# memory_percent‚94.1633939743042" proc.memory.rssMB‚ 2621.6484375 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 30.63671875$ disk./.usageGB‚301.9990921020508 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 network.recv‚ 2.036066e+06 disk.sda2.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚22551.54345703125( proc.memory.availableMB‚ 956.26953125 cpu‚25.317288138644965wų[“È‚(:ˆ ú­æÔ”Žïü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.409' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0h=„ pÉ‚(:å ú­æÔü¸üþ disk.drbd2266.out‚0$ memory_percent‚94.16189193725586 disk.rbd0.out‚0( disk.nvme0c0n1.out‚22601.62548828125& proc.memory.availableMB‚ 956.515625 proc.cpu.threads‚17 network.recv‚ 2.043128e+06# disk.dm-0.out‚22601.62548828125 disk.sda.in‚ 0.01171875 disk.dm-1.out‚ 218.9453125 disk.drbd2265.out‚0 cpu‚25.31655838941929 network.sent‚ 1.7044072e+07 disk.sda.out‚ 218.9453125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.rbd1.in‚0& disk.nvme0n1.out‚22601.62548828125 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2266.in‚0$ disk./.usageGB‚301.9922409057617 disk.sda2.out‚0 disk.rbd1.out‚ 30.69921875 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 218.9453125" proc.memory.rssMB‚ 2621.6484375* proc.memory.percent‚16.001272201538086* disk./.usagePercent‚20.266952675553174 disk.drbd1716.out‚0 disk.drbd2080.out‚0}mÔ:2Ê- _wandbruntime‚5677  _runtime‚5677¥çÅË¿ trainer/global_step‚7199 # _timestamp‚1.78845056208326e+09 _runtime‚5677.554825824 _step‚587 & lr-AdamW/pg1‚5.333540908032184e-05 & lr-AdamW/pg2‚5.333540908032184e-05…¢†ÊÌÄ & lr-AdamW/pg1‚5.333540908032184e-05 & lr-AdamW/pg2‚5.333540908032184e-05 trainer/global_step‚7199 # _timestamp‚1.78845056208326e+09 _runtime‚5677.554825824 _step‚587ˆ¬¾ 2Í- _wandbruntime‚5677  _runtime‚56771`ëå'ÎßO UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 135675755 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚462150037248512 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 200146944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6313238642688 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚412437616942646 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚412438288419762 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚462150237395456 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 239982403 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚406442957659136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚455836798605824 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 135655717 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚417360233260544 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176206848 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚423377038233600 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 104326688 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _runtime‚5677.555738086 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 135675755 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 104326686 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚7199 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6017461335552 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 135655717 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚588 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6016804973056 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5994659283510 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚423377712599040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6016804973056 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚423377038233600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 104326686 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚34289387 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚34288937 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚34289387 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚34288937 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚412438288419762 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚462150237395456 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 239982403 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚406442957659136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚455836798605824 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 135655717 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚417360233260544 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 176206848 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚423377038233600 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 104326688 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _runtime‚5677.555738086 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 135675755 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 104326686 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 trainer/global_step‚7199 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6017461335552 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 135655717 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚588 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6016804973056 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 5994659283510 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚423377712599040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6016804973056 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚423377038233600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025ÌÉém*2Ð- _wandbruntime‚5678  _runtime‚5678W~ÑçM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚417416604302336 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚412494008985494 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚412493337508170 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 240034939 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6018280776704 T'elMo#EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚60872228 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6017624400896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 240014873 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚589 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚446 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 5678.23018349 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚95165660 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚455898449343488 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6314119564800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚455898473283584 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 240014873 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6017624400896 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚444 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 104340702 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 135694235 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 240034939 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 104340704 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5995475718474 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚417416586299392 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚462212568908288 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚417416586299392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 135674171 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 $ _timestamp‚1.788450562758067e+09 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚423434210700288 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 trainer/global_step‚7199 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 104340702 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚406497878829056 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 135694235 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚417416604302336 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 104340704 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚34293432 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚406497861789696 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚95166106 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚423434885079040 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844¤’Inò&ÒìM F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚417416604302336 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚412494008985494 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚412493337508170 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 240034939 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6018280776704 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚60872228 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6017624400896 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 240014873 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚589 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚446 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 5678.23018349 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚95165660 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚455898449343488 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6314119564800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚455898473283584 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 240014873 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6017624400896 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚444 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 104340702 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 135694235 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 240034939 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 104340704 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 5995475718474 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚417416586299392 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚462212568908288 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚417416586299392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 135674171 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 $ _timestamp‚1.788450562758067e+09 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚423434210700288 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 trainer/global_step‚7199 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 104340702 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚406497878829056 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 135694235 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚417416604302336 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 104340704 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚34293432 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚406497861789696 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚95166106 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚423434885079040ͽžô`2Ó- _wandbruntime‚5678  _runtime‚5678}…jœWÔÑ 2loss/SRqqt1/assignment_loss‚1.7790495702887283 2loss/SRqqt3/assignment_loss‚1.3270234147891065 /loss/FRt3/detection_loss‚1.7469610363758843 /loss/SRqqt1/detection_loss‚2.61563304066658 1loss/SRqqt3/detection_loss‚1.0778463422461892 1loss/SRqqt4/detection_loss‚1.7026844588970695 % _timestamp‚1.7884505627596626e+09 0loss/FBt1/assignment_loss‚0.5089003576618519 1loss/FBt3/assignment_loss‚0.25858587776182995 /loss/FRt1/detection_loss‚3.1752705574035645 .loss/FBt1/detection_loss‚2.406015313097409 .loss/FBt4/detection_loss‚2.536731575216566 _runtime‚5678.231204828 2loss/SRqqt2/assignment_loss‚1.5674940527975558 0loss/FBt4/assignment_loss‚0.9605258415852275 .loss/FRt4/detection_loss‚2.090086948202581 /loss/FRt1/assignment_loss‚4.250285228093465 /loss/FRt3/assignment_loss‚1.492523647587875 0loss/FBt2/assignment_loss‚0.5113106342878685 %loss/total_loss‚44.36321019682268 _step‚590 2loss/SRqqt4/assignment_loss‚1.4537735245432015 epoch‚5 /loss/FRt2/assignment_loss‚2.932842178671968 /loss/FRt2/detection_loss‚2.1158619020499434 0loss/SRqqt2/detection_loss‚1.864964980483055 trainer/global_step‚7199 0loss/FRt4/assignment_loss‚2.8248658241057885 /loss/FBt2/detection_loss‚2.3781975453550164 /loss/FBt3/detection_loss‚0.7857763446543528Ò9“Ð\ÕÖ /loss/FRt2/assignment_loss‚2.932842178671968 /loss/FRt2/detection_loss‚2.1158619020499434 0loss/SRqqt2/detection_loss‚1.864964980483055 trainer/global_step‚7199 0loss/FRt4/assignment_loss‚2.8248658241057885 /loss/FBt2/detection_loss‚2.3781975453550164 /loss/FBt3/detection_loss‚0.7857763446543528 2loss/SRqqt1/assignment_loss‚1.7790495702887283 2loss/SRqqt3/assignment_loss‚1.3270234147891065 /loss/FRt3/detection_loss‚1.7469610363758843 /loss/SRqqt1/detection_loss‚2.61563304066658 1loss/SRqqt3/detection_loss‚1.0778463422461892 1loss/SRqqt4/detection_loss‚1.7026844588970695 % _timestamp‚1.7884505627596626e+09 0loss/FBt1/assignment_loss‚0.5089003576618519 1loss/FBt3/assignment_loss‚0.25858587776182995 /loss/FRt1/detection_loss‚3.1752705574035645 .loss/FBt1/detection_loss‚2.406015313097409 .loss/FBt4/detection_loss‚2.536731575216566 _runtime‚5678.231204828 2loss/SRqqt2/assignment_loss‚1.5674940527975558 0loss/FBt4/assignment_loss‚0.9605258415852275 .loss/FRt4/detection_loss‚2.090086948202581 /loss/FRt1/assignment_loss‚4.250285228093465 /loss/FRt3/assignment_loss‚1.492523647587875 0loss/FBt2/assignment_loss‚0.5113106342878685 %loss/total_loss‚44.36321019682268 _step‚590 2loss/SRqqt4/assignment_loss‚1.4537735245432015 epoch‚5Î:¥$¶žÖ‚(:“ ‰®æÔÑÄ™ý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚124.866' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.973200000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¶(Lôjׂ(:ß ‰®æԢ癀* disk./.usagePercent‚20.270310726379805 disk.drbd1716.out‚0 proc.cpu.threads‚17 disk.sda2.in‚0 disk.rbd1.in‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 network.sent‚ 1.7049216e+07 disk.rbd0.out‚0 disk.sda2.out‚0 disk.dm-1.out‚ 219.609375$ memory_percent‚94.16587352752686 cpu‚25.315891112756184 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0( proc.memory.availableMB‚ 955.86328125 disk.nvme0c0n1.in‚0.078125! disk.dm-0.out‚22672.701171875$ disk.nvme0n1.out‚22672.701171875 disk.drbd2265.out‚0" proc.memory.rssMB‚ 2621.6484375 disk.dm-1.in‚ 0.01171875 network.recv‚ 2.046524e+06$ disk./.usageGB‚302.0422782897949& disk.nvme0c0n1.out‚22672.701171875 disk.sda1.out‚0 disk.drbd2080.out‚0* proc.memory.percent‚16.001272201538086 disk.sda.in‚ 0.01171875 disk.sda.out‚ 219.609375 disk.drbd2265.in‚0 disk.sda3.out‚ 219.609375 disk.rbd1.out‚ 30.69921875 disk.drbd1716.in‚0yÍužØ‚(:“ ˜®æÔã΃ý gpu.0.gpu‚31.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚170.842' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚34.168400000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ö`ƒkÙ‚(:à ˜®æÔ«ëýþ disk.sda1.in‚0& disk.nvme0c0n1.out‚22722.072265625 disk.drbd2266.in‚0! disk.dm-0.out‚22722.072265625$ memory_percent‚94.16589736938477 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0" proc.memory.rssMB‚ 2621.6484375 proc.cpu.threads‚17* disk./.usagePercent‚20.270856534442803 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.sda.out‚ 220.296875 disk.rbd1.out‚ 30.7314453125 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 220.296875 disk.sda1.out‚0 network.sent‚ 1.7157252e+07 network.recv‚ 2.052709e+06 disk.drbd2265.in‚0* proc.memory.percent‚16.001272201538086% disk./.usageGB‚302.05041122436523$ disk.nvme0n1.out‚22722.072265625 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0 disk.drbd2080.out‚0& proc.memory.availableMB‚ 955.859375 cpu‚25.315162503968434 disk.rbd1.in‚0 disk.dm-1.out‚ 220.296875 disk.drbd1716.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875ÇF"2Ú- _wandbruntime‚5714  _runtime‚5714ð<µÖÉÛà trainer/global_step‚7249 % _timestamp‚1.7884505988479867e+09 _runtime‚5714.319562768 _step‚591 ' lr-AdamW/pg1‚5.2433541363165496e-05 ' lr-AdamW/pg2‚5.2433541363165496e-05<ºÓ=ÎÜÈ ' lr-AdamW/pg1‚5.2433541363165496e-05 ' lr-AdamW/pg2‚5.2433541363165496e-05 trainer/global_step‚7249 % _timestamp‚1.7884505988479867e+09 _runtime‚5714.319562768 _step‚591Ï<À,82Ý- _wandbruntime‚5714  _runtime‚5714åùYLÞßO YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚420177885202944 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6057790778880 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 105027486 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 241609255 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚61272462 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚465275845859840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚409188181226496 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚458919335489024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6058447141376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6035494622918 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 105027488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6356329098240 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚426235675981824 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚420177903205888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚426236350347264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚34516654 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requ¹œ*Y™ ested_bytes.all.allocated‚415224330287170 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚426235675981824 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 136581769 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6058447141376 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6057790778880 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 241629295 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 105027488 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 105027486 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚426236350347264 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884505988486485e+09 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6035494622918 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 105027488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6356329098240 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚426235675981824 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚420177903205888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚426236350347264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚34516654 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚415224330287170 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚426235675981824 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 136581769 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6058447141376 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6057790778880 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 241629295 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 105027488 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 105027486 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚426236350347264 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884505988486485e+09 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚61272462 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚465275845859840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚409188181226496 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚458919335489024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0ÐxVÑ42à- _wandbruntime‚5714  _runtime‚5714°Ø¨‡î&áèM JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚5714.999180229 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚420234238241792 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 136620287 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚95801272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚34521084 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 105041502 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚465338362200576 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚7249 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚420234238241792 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚61280628 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 136600223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 241661791 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 105041504 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 $ _timestamp‚1.788450599527004e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚95801714 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 105041504 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6357182131712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _step‚593 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6058610206720 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚420234256244736 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚458980986226688 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚409243085357056 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6058610206720 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚34520644 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6036965495846 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚465338168358400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚415279379375578 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚426293522827264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚420234256244736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 241661791 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 136620287 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521t+¢:%âíM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 241641725 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚409243068317696 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6058610206720 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚420234256244736 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚458980986226688 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚409243085357056 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6058610206720 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚34520644 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6036965495846 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚465338168358400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚415279379375578 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚426293522827264 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚420234256244736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 241661791 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 136620287 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚5714.999180229 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚420234238241792 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 136620287 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚95801272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚34521084 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 105041502 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚465338362200576 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 trainer/global_step‚7249 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚420234238241792 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚61280628 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 136600223 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 241661791 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 105041504 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 $ _timestamp‚1.788450599527004e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚95801714 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 105041504 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6357182131712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _step‚593 EDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚417250158841938 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914392 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚428344277631488 OADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚ 242661890 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚467462387042304 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 SEDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚ 242677869 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚461083500119040 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚411192096094208 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 _step‚595 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚417249650888186 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 141381120 RDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚ 105627060 bNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚422264339811328 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚428344788464640 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚153 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 242677869 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 137050807 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚422264321808384 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 117441024 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚ 137034830 trainer/global_step‚7253 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 \JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚ 6079955823104 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚61631996 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚411192079054848 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚96393369 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 6058062747730 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 jVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.allocated‚461083524059136 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 6378745542144 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 6057571833338 LCDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak‚2605 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚ 105627062 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚422264321808384 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15977 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 137034830 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚151 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 BDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚417250158841938 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914392 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚428344277631488 OADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚ 242661890 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚467462387042304 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 SEDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚ 242677869 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚461083500119040 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚411192096094208 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 _step‚595 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚417249650888186 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 141381120 RDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚ 105627060 bNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚422264339811328 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚428344788464640 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚153 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 242677869 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 137050807 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚422264321808384 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 117441024 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚ 137034830 trainer/global_step‚7253 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 \JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚ 6079955823104 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚61631996 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚411192079054848 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚96393369 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 6058062747730 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 jVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.allocated‚461083524059136 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 6378745542144 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 6057571833338 LCDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak‚2605 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚ 105627062 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚422264321808384 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15977 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 137034830 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚151 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928Óù©032ñ- _wandbruntime‚5767  _runtime‚5767M0“æíòÐR UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914392 XIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 MEDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.current‚414 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 6080320949760 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 117441024 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_validation_batch_end/oversize_segments.current‚0 cODeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.allocated‚411227389853696 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 TFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚ 105637866 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚34765924 PBDeviceStatsMonitor.on_validation_batch_end/active.large_pool.freed‚ 105637866 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚428381983620096 aMDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.allocâ d‘i#ated‚467500627902464 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.peak‚ 868220928 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 242695977 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚422301151837184 aODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.allocated‚ 6080813779968 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚152 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚422301151837184 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 UADeviceStatsMonitor.on_validation_batch_end/active_bytes.all.freed‚428381472786944 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 6379228109824 bPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚ 6379110668800 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 PCDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed‚96404272 % _timestamp‚1.7884506518270543e+09 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.current‚ 510833152 VGDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak‚ 9148669952 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.freed‚34765772 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.freed‚0 XDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚428381472786944 ]KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚ 6057935442746 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚ 242679998 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 \NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 I;DeviceStatsMonitor.on_validation_batch_end/active.all.freed‚ 242679998 OEDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.peak‚24128 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 MEDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.current‚154 JDDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak‚0 _KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.freed‚411227372814336 _runtime‚5767.298998013 RCDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 242695977 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 [LDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚61638502 TFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚ 105637868 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚422301169840128 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 337538560 hTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚461121399792640 XDDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.freed‚417285308257082 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 XJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚ 105637868 dPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.freed‚461121375852544 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 ]KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚ 6080320949760 XJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚ 137058109 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 MGDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.current‚0 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24401 TFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚ 137058109 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚422301169840128 PBDeviceStatsMonitor.on_validation_batch_end/active.small_pool.freed‚ 137042132 TFDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.current‚ 510833152 TGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚96404426 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.peak‚889 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977¬?…d[)óÕR [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914392 XIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 MEDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.current‚414 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 6080320949760 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 117441024 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_validation_batch_end/oversize_segments.current‚0 cODeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.allocated‚411227389853696 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 TFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚ 105637866 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚34765924 PBDeviceStatsMonitor.on_validation_batch_end/active.large_pool.freed‚ 105637866 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚428381983620096 aMDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.allocated‚467500627902464 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.peak‚ 868220928 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 242695977 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚422301151837184 aODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.allocated‚ 6080813779968 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚152 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚422301151837184 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 UADeviceStatsMonitor.on_validation_batch_end/active_bytes.all.freed‚428381472786944 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 6379228109824 bPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚ 6379110668800 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 PCDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed‚96404272 % _timestamp‚1.7884506518270543e+09 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.current‚ 510833152 VGDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak‚ 9148669952 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.freed‚34765772 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.freed‚0 XDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚428381472786944 ]KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚ 6057935442746 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚ 242679998 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 \NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 I;DeviceStatsMonitor.on_validation_batch_end/active.all.freed‚ 242679998 OEDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.peak‚24128 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 MEDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.current‚154 JDDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak‚0 _KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.freed‚411227372814336 _runtime‚5767.298998013 RCDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 242695977 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 [LDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚61638502 TFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚ 105637868 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚422301169840128 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 337538560 hTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚461121399792640 XDDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.freed‚417285308257082 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 XJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚ 105637868 dPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.freed‚461121375852544 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 ]KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚ 6080320949760 XJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚ 137058109 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 MGDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.current‚0 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24401 TFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚ 137058109 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚422301169840128 PBDeviceStatsMonitor.on_validation_batch_end/active.small_pool.freed‚ 137042132 TFDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.current‚ 510833152 TGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚96404426 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.peak‚889 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680ÔÏøõ‘ô‚(:† ã®æÔúÏ·ý gpu.0.gpu‚52.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.68' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.536 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0°)¢oõ‚(:ä ã®æÔÆðÚ€ disk.dm-1.out‚ 222.8203125 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.sda1.in‚0 proc.memory.rssMB‚2642.875* proc.memory.percent‚16.130828857421875 network.sent‚ 1.7288571e+07 network.recv‚ 2.075333e+06$ disk./.usageGB‚302.1291618347168& disk.nvme0n1.out‚22983.83447265625 disk.sda1.out‚0 disk.rbd1.out‚ 31.4580078125 disk.sda.out‚ 222.8203125 proc.cpu.threads‚17 disk.sda3.out‚ 222.8203125 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875 disk.sda2.in‚0$ memory_percent‚94.28942203521729) disk./.usagePercent‚20.27614155397966 disk.drbd2080.in‚0 disk.drbd2266.in‚0# disk.dm-0.out‚22983.83447265625 disk.sda2.out‚0 disk.drbd1716.in‚0( disk.nvme0c0n1.out‚22983.83447265625 disk.sda3.in‚ 0.01171875 cpu‚25.35970279602701 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 935.62109375 disk.nvme0n1.in‚0.078125 z62ö- _wandbruntime‚5780  _runtime‚5780 `áà,÷Þ¦ >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 7 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.1666666716337204 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3928571343421936 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9246032238006592 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9973471164703369 particle/accuracy_1_of_4‚1 ( Purity/*FRt3SRqqt2FBt/FBt_purity‚0.5 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9735631346702576 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9433333873748779 ;"Purity/1FRt2SRqqt*FBt/event_purity‚0.000933706818614155 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5717592835426331 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1723509281873703 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 8"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.582143247127533 particle/accuracy_2_of_7‚1 & Purity/*FRt*SRqqt4FBt/FRt_purity‚0 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5771946907043457 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 particle/accuracy_1_of_1‚1 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9921421408653259 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 7 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.2298850566148758 particle/accuracy_5_of_7‚1 particle/accuracy_6_of_7‚1 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9877151250839233 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5561336874961853 *jet/accuracy_1_of_2‚0.7707348465919495 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.14605925977230072 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.6111111044883728 7 Purity/*FRt3SRqqt*FBt/FBt_purity‚0.9821428656578064 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 *jet/accuracy_2_of_6‚0.4871794581413269 9"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.8999999761581421 -"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.8125 ) Purity/4FRt*SRqqt*FBt/FRt_purity‚0.25 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1348167210817337 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.11419752240180969 particle/accuracy_3_of_4‚1 9"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.4945533871650696 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5019091963768005 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.0625 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4410330653190613 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.6212120652198792 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9869614839553833 8"Purity/0FRt1SRqqt0FBt/event_purity‚0.563859760761261 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2301587164402008 particle/accuracy_4_of_5‚1 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.20367415249347687 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5784652829170227 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.40740740299224854 & Purity/1FRt*SRqqt4FBt/FRt_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14674919843673706 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9888995289802551 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ( Purity/4FRt1SRqqt0FBt/FRt_purity‚0.5 _step‚597 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.01695011369884014 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 % _timestamp‚1.7884506647704895e+09 7 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.2356473058462143 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.004576500505208969 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4463600516319275 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 particle/accuracy_1_of_5‚1 6 Purity/1FRt3SRqqt*FBt/FBt_purity‚0.970588207244873 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.2081446349620819 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.022270116955041885 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5159480571746826 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.13333335518836975 particle/accuracy_5_of_6‚1 particle/accuracy_3_of_7‚1 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.2558025121688843 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5703415870666504 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.2666666805744171 ,"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.375 8 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.33212119340896606 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.0555555559694767 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 jet/accuracy_5_of_6‚0 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.3788255751132965 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.13463522493839264 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9688491821289062 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.6515151262283325 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 +jet/accuracy_2_of_4‚0.36326199769973755 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9761301875114441 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 particle/accuracy_4_of_6‚1 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 6 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.996132493019104 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.019790880382061005 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 ( Purity/1FRt3SRqqt2FBt/FBt_purity‚0.5 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9351851940155029 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.024372760206460953 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.18809741735458374 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5966444611549377 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.03546099364757538 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9947453737258911 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.1367490589618683 * Purity/1FRt0SRqqt3FBt/FRt_purity‚0.625 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ;"Purity/0FRt*SRqqt3FBt/event_purity‚0.048245616257190704 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.14605404436588287 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5468406677246094 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9663195610046387 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.14991843700408936 *jet/accuracy_2_of_5‚0.4603494703769684 9"Purity/0FRt0SRqqt3FBt/event_purity‚0.0572916679084301 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.7708333134651184 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.17950449883937836 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 particle/accuracy_1_of_6‚1 9"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.4974946081638336 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ,particle/sensitivity‚0.30242466926574707 jet/accuracy_6_of_6‚0 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5184924602508545 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.13675829768180847 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.24607940018177032 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 7validation_average_jet_accuracy‚0.45126500725746155 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.30000001192092896 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4566666781902313 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.1666666716337204 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/event_purity‚¸†ô’…f0.6490152478218079 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9876196384429932 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 8"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.572918176651001 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.3895348906517029 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5787188410758972 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9931098222732544 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9869856834411621 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 'particle/f_score‚0.3954252302646637 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.18654529750347137 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.04434523731470108 particle/accuracy_3_of_5‚1 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 _runtime‚5780.244171921 * Purity/2FRt1SRqqt2FBt/FRt_purity‚0.375 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 +particle/specificity‚0.9581311345100403 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.3950617015361786 particle/accuracy_3_of_6‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.30683937668800354 9"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.4596051871776581 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_1‚0.5135550498962402 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.40820062160491943 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 jet/accuracy_1_of_7‚1 7 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.9047619104385376 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9625256061553955 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.49367600679397583 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9857677817344666 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9525861144065857 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 8"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.430029958486557 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 *jet/accuracy_2_of_2‚0.1100042536854744 ("Purity/*FRt3SRqqt2FBt/SRqqt_purity‚1 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.9074074625968933 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 *"Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0.5 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.25980833172798157 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0694444477558136 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 *"Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0.5 5 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.98326176404953 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5116869211196899 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 6 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.903508722782135 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 *jet/accuracy_1_of_5‚0.9614119529724121 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9789075255393982 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9778879284858704 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 6 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.292076975107193 :"Purity/2FRt*SRqqt*FBt/event_purity‚0.00895592663437128 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 jet/accuracy_5_of_7‚0 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.45495903491973877 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1827298104763031 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_5‚1 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3611111044883728 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5589056611061096 8"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.566422700881958 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5495240688323975 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0694444477558136 jet/accuracy_6_of_7‚0 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.3732639253139496 *jet/accuracy_1_of_4‚0.9239152669906616 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9929894208908081 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.6039600372314453 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6300190687179565 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5747840404510498 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5843963623046875 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.6176470518112183 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5341102480888367 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.6315789222717285 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 +jet/accuracy_3_of_3‚0.02292102389037609 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 +jet/accuracy_4_of_6‚0.01923076994717121 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.07629923522472382 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0515873022377491 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.09956255555152893 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.14089764654636383 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 particle/accuracy_1_of_3‚1 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.11539915204048157 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5918482542037964 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.4653759002685547 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.5866667032241821 ("Purity/1FRt3SRqqt2FBt/SRqqt_purity‚1 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.02421383559703827 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ,"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.375 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.3462797701358795 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.40625 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.3776455223560333 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.18776492774486542 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1723509281873703 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,jet/accuracy_3_of_4‚0.044920098036527634 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.1945943385362625 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 particle/accuracy_2_of_2‚1 (particle/accuracy‚0.8557631969451904 particle/accuracy_6_of_6‚1 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.1666666716337204 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 8"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.515507698059082 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.3371207118034363 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5726190209388733 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.30000001192092896 particle/accuracy_1_of_7‚1 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.1074616089463234 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.19979068636894226 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 epoch‚5 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.18484848737716675 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.3854166567325592 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.4166666567325592 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.4919762909412384 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.14163483679294586 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 jet/accuracy_5_of_5‚0 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.4166666567325592 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09394034743309021 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.040358517318964005 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5011999607086182 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 jet/accuracy_1_of_6‚1 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.538215696811676 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6793789863586426 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.6388888955116272 ,jet/accuracy_4_of_5‚0.010816692374646664 particle/accuracy_5_of_5‚1 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.47112077474594116 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.12226216495037079 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 + Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9375 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5574546456336975 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6264368295669556 ( Purity/4FRt1SRqqt*FBt/FRt_purity‚0.5 )loss/val_total_loss‚46.31566687502204 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 +jet/accuracy_3_of_6‚0.07692307978868484 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.29667189717292786 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9896175265312195 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 8"Purity/0FRt0SRqqt*FBt/event_purity‚0.812539279460907 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0014430014416575432 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4376918077468872 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.1061413586139679 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.11590037494897842 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 *"Purity/0FRt2SRqqt3FBt/SRqqt_purity‚0.5 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3790300488471985 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.49824947118759155 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.03507956489920616 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9166666865348816 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 particle/accuracy_7_of_7‚0.5 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 6 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.190476194024086 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9887279868125916 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3611111044883728 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5401400327682495 8"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.585010290145874 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_4_of_4‚1 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.34375 jet/accuracy_7_of_7‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 8"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.563859760761261 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08440754562616348 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 particle/accuracy_2_of_6‚1 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9929742813110352 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15144604444503784 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.18209877610206604 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17445887625217438 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.31959834694862366 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5442838072776794 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5519475936889648 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9910547137260437 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.44595956802368164 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 )jet/accuracy_1_of_3‚0.881702184677124 particle/accuracy_1_of_2‚1 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2499999701976776 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.5034613013267517 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5492150783538818 7 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.1307646781206131 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5454860925674438 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 ,jet/accuracy_4_of_4‚0.007123504299670458 particle/accuracy_3_of_3‚1 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.45885854959487915 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.01742386445403099 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 +jet/accuracy_3_of_5‚0.07523681223392487 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4479166865348816 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4000000059604645 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 trainer/global_step‚7253 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.6052632331848145 6 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.289814829826355 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.3333333432674408 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5910303592681885 ;"Purity/2FRt0SRqqt2FBt/event_purity‚0.032894738018512726 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.7708333134651184 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.24525824189186096 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5924475789070129 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.022503063082695007 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.18826472759246826 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15338686108589172 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.13362573087215424 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.024820201098918915 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5450372695922852 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9760233759880066 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.13649298250675201 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 jet/accuracy_2_of_7‚0.5 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 *"Purity/*FRt2SRqqt3FBt/SRqqt_purity‚0.5 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.07973955571651459 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 7 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.1228875070810318 ;"Purity/2FRt*SRqqt2FBt/event_purity‚0.011627906933426857 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 ) Purity/4FRt*SRqqt0FBt/FRt_purity‚0.25 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3812226355075836 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.4089912176132202 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 *jet/accuracy_2_of_3‚0.2506849765777588 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4963393807411194 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.23984913527965546 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.2596839964389801 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5506432056427002 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9107142686843872 jet/accuracy_3_of_7‚0 particle/accuracy_4_of_7‚1 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.04034871980547905 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.23957157135009766 , Purity/1FRt3SRqqt1FBt/FRt_purity‚0.28125 jet/accuracy_4_of_7‚0 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9784632325172424 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.2127343863248825 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5395154356956482 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.30277779698371887 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5535714030265808 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.012188208289444447 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5612605810165405 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 7 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.1270114928483963 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.3388392925262451 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.17909322679042816 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3269171416759491 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.18174602091312408 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5437366366386414 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9795260429382324 :"Purity/1FRt1SRqqt2FBt/event_purity‚0.02256944589316845 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5634921193122864 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9895116090774536 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.2462320476770401 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9172577261924744 Ìâ1mø㦠7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.2558025121688843 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5703415870666504 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.2666666805744171 ,"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.375 8 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.33212119340896606 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.0555555559694767 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 jet/accuracy_5_of_6‚0 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.3788255751132965 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.13463522493839264 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9688491821289062 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.6515151262283325 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 +jet/accuracy_2_of_4‚0.36326199769973755 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9761301875114441 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 particle/accuracy_4_of_6‚1 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 6 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.996132493019104 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.019790880382061005 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 ( Purity/1FRt3SRqqt2FBt/FBt_purity‚0.5 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9351851940155029 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.024372760206460953 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.18809741735458374 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5966444611549377 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.03546099364757538 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9947453737258911 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.1367490589618683 * Purity/1FRt0SRqqt3FBt/FRt_purity‚0.625 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ;"Purity/0FRt*SRqqt3FBt/event_purity‚0.048245616257190704 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.14605404436588287 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5468406677246094 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9663195610046387 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.14991843700408936 *jet/accuracy_2_of_5‚0.4603494703769684 9"Purity/0FRt0SRqqt3FBt/event_purity‚0.0572916679084301 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.7708333134651184 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.17950449883937836 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 particle/accuracy_1_of_6‚1 9"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.4974946081638336 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ,particle/sensitivity‚0.30242466926574707 jet/accuracy_6_of_6‚0 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5184924602508545 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.13675829768180847 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.24607940018177032 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 7validation_average_jet_accuracy‚0.45126500725746155 :"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.30000001192092896 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4566666781902313 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.1666666716337204 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6490152478218079 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9876196384429932 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 8"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.572918176651001 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.3895348906517029 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5787188410758972 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9931098222732544 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9869856834411621 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 'particle/f_score‚0.3954252302646637 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.18654529750347137 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.0443456Ì…ýy23731470108 particle/accuracy_3_of_5‚1 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 _runtime‚5780.244171921 * Purity/2FRt1SRqqt2FBt/FRt_purity‚0.375 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 +particle/specificity‚0.9581311345100403 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.3950617015361786 particle/accuracy_3_of_6‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.30683937668800354 9"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.4596051871776581 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_1‚0.5135550498962402 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.40820062160491943 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 jet/accuracy_1_of_7‚1 7 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.9047619104385376 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9625256061553955 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.49367600679397583 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9857677817344666 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9525861144065857 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 8"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.430029958486557 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 *jet/accuracy_2_of_2‚0.1100042536854744 ("Purity/*FRt3SRqqt2FBt/SRqqt_purity‚1 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.9074074625968933 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 *"Purity/4FRt*SRqqt0FBt/SRqqt_purity‚0.5 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.25980833172798157 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0694444477558136 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 *"Purity/4FRt*SRqqt*FBt/SRqqt_purity‚0.5 5 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.98326176404953 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5116869211196899 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 6 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.903508722782135 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 *jet/accuracy_1_of_5‚0.9614119529724121 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9789075255393982 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9778879284858704 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚0 6 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.292076975107193 :"Purity/2FRt*SRqqt*FBt/event_purity‚0.00895592663437128 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 jet/accuracy_5_of_7‚0 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.45495903491973877 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1827298104763031 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_5‚1 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3611111044883728 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5589056611061096 8"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.566422700881958 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5495240688323975 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0694444477558136 jet/accuracy_6_of_7‚0 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.3732639253139496 *jet/accuracy_1_of_4‚0.9239152669906616 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9929894208908081 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.6039600372314453 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6300190687179565 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5747840404510498 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5843963623046875 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.6176470518112183 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5341102480888367 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.6315789222717285 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 +jet/accuracy_3_of_3‚0.02292102389037609 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 +jet/accuracy_4_of_6‚0.01923076994717121 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.07629923522472382 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0515873022377491 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.09956255555152893 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.14089764654636383 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 particle/accuracy_1_of_3‚1 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.11539915204048157 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5918482542037964 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.4653759002685547 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.5866667032241821 ("Purity/1FRt3SRqqt2FBt/SRqqt_purity‚1 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.02421383559703827 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ,"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.375 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.3462797701358795 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.40625 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.3776455223560333 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.18776492774486542 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1723509281873703 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,jet/accuracy_3_of_4‚0.044920098036527634 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.1945943385362625 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 particle/accuracy_2_of_2‚1 (particle/accuracy‚0.8557631969451904 particle/accuracy_6_of_6‚1 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.1666666716337204 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 8"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.515507698059082 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.3371207118034363 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5726190209388733 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 :"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.30000001192092896 particle/accuracy_1_of_7‚1 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.1074616089463234 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.19979068636894226 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 epoch‚5 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.18484848737716675 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.3854166567325592 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.4166666567325592 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.4919762909412384 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.14163483679294586 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 jet/accuracy_5_of_5‚0 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.4166666567325592 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09394034743309021 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.040358517318964005 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5011999607086182 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 jet/accuracy_1_of_6‚1 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.538215696811676 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6793789863586426 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.6388888955116272 ,jet/accuracy_4_of_5‚0.010816692374646664 particle/accuracy_5_of_5‚1 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.47112077474594116 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.12226216495037079 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 + Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9375 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5574546456336975 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6264368295669556 ( Purity/4FRt1SRqqt*FBt/FRt_purity‚0.5 )loss/val_total_loss‚46.31566687502204 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 +jet/accuracy_3_of_6‚0.07692307978868484 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.29667189717292786 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9896175265312195 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 8"Purity/0FRt0SRqqt*FBt/event_purity‚0.812539279460907 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0014430014416575432 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4376918077468872 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.1061413586139679 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.11590037494897842 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 *"Purity/0FRt2SRqqt3FBt/SRqqt_purity‚0.5 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3790300488471985 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.49824947118759155 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.03507956489920616 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9166666865348816 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 particle/accuracy_7_of_7‚0.5 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 6 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.190476194024086 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9887279868125916 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3611111044883728 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5401400327682495 8"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.585010290145874 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_4_of_4‚1 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.34375 jet/accuracy_7_of_7‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 8"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.563859760761261 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08440754562616348 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 particle/accuracy_2_of_6‚1 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9929742813110352 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15144604444503784 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.18209877610206604 & Purity/1FRt0SRqqt4FBt/FRt_purity‚0 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17445887625217438 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.31959834694862366 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5442838072776794 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5519475936889648 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9910547137260437 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.44595956802368164 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 )jet/accuracy_1_of_3‚0.881702184677124 particle/accuracy_1_of_2‚1 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2499999701976776 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.5034613013267517 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5492150783538818 7 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.1307646781206131 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5454860925674438 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 ,jet/accuracy_4_of_4‚0.007123504299670458 particle/accuracy_3_of_3‚1 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.45885854959487915 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.01742386445403099 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 +jet/accuracy_3_of_5‚0.07523681223392487 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4479166865348816 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4000000059604645 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 trainer/global_step‚7253 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.6052632331848145 6 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.289814829826355 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.3333333432674408 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5910303592681885 ;"Purity/2FRt0SRqqt2FBt/event_purity‚0.032894738018512726 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.7708333134651184 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.24525824189186096 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5924475789070129 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.022503063082695007 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.18826472759246826 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15338686108589172 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.13362573087215424 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.024820201098918915 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5450372695922852 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9760233759880066 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.13649298250675201 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 jet/accuracy_2_of_7‚0.5 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 *"Purity/*FRt2SRqqt3FBt/SRqqt_purity‚0.5 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.07973955571651459 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 7 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.1228875070810318 ;"Purity/2FRt*SRqqt2FBt/event_purity‚0.011627906933426857 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 ) Purity/4FRt*SRqqt0FBt/FRt_purity‚0.25 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3812226355075836 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.4089912176132202 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 *jet/accuracy_2_of_3‚0.2506849765777588 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4963393807411194 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.23984913527965546 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.2596839964389801 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5506432056427002 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9107142686843872 jet/accuracy_3_of_7‚0 particle/accuracy_4_of_7‚1 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.04034871980547905 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.23957157135009766 , Purity/1FRt3SRqqt1FBt/FRt_purity‚0.28125 jet/accuracy_4_of_7‚0 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9784632325172424 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.2127343863248825 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5395154356956482 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.30277779698371887 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5535714030265808 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.012188208289444447 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5612605810165405 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 7 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.1270114928483963 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.3388392925262451 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.17909322679042816 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3269171416759491 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.18174602091312408 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5437366366386414 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9795260429382324 :"Purity/1FRt1SRqqt2FBt/event_purity‚0.02256944589316845 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5634921193122864 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9895116090774536 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.2462320476770401 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9172577261924744 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 7 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.1666666716337204 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3928571343421936 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9246032238006592 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9973471164703369 particle/accuracy_1_of_4‚1 ( Purity/*FRt3SRqqt2FBt/FBt_purity‚0.5 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9735631346702576 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9433333873748779 ;"Purity/1FRt2SRqqt*FBt/event_purity‚0.000933706818614155 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5717592835426331 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1723509281873703 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 8"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.582143247127533 particle/accuracy_2_of_7‚1 & Purity/*FRt*SRqqt4FBt/FRt_purity‚0 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5771946907043457 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 particle/accuracy_1_of_1‚1 & Purity/*FRt0SRqqt4FBt/FRt_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9921421408653259 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 7 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.2298850566148758 particle/accuracy_5_of_7‚1 particle/accuracy_6_of_7‚1 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9877151250839233 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5561336874961853 *jet/accuracy_1_of_2‚0.7707348465919495 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.14605925977230072 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.6111111044883728 7 Purity/*FRt3SRqqt*FBt/FBt_purity‚0.9821428656578064 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 *jet/accuracy_2_of_6‚0.4871794581413269 9"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.8999999761581421 -"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.8125 ) Purity/4FRt*SRqqt*FBt/FRt_purity‚0.25 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1348167210817337 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.11419752240180969 particle/accuracy_3_of_4‚1 9"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.4945533871650696 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5019091963768005 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.0625 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4410330653190613 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.6212120652198792 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9869614839553833 8"Purity/0FRt1SRqqt0FBt/event_purity‚0.563859760761261 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2301587164402008 particle/accuracy_4_of_5‚1 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.20367415249347687 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5784652829170227 :"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.40740740299224854 & Purity/1FRt*SRqqt4FBt/FRt_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14674919843673706 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9888995289802551 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ( Purity/4FRt1SRqqt0FBt/FRt_purity‚0.5 _step‚597 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.01695011369884014 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚0 % _timestamp‚1.7884506647704895e+09 7 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.2356473058462143 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.004576500505208969 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4463600516319275 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 particle/accuracy_1_of_5‚1 6 Purity/1FRt3SRqqt*FBt/FBt_purity‚0.970588207244873 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.2081446349620819 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.022270116955041885 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5159480571746826 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.13333335518836975 particle/accuracy_5_of_6‚1 particle/accuracy_3_of_7‚1ÕY„TΓù‚(:ˆ ò®æÔßù«ý gpu.0.gpu‚56.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚154.808' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚30.9616 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ø~u€[ú‚(:ç ò®æÔÏÉŒ€ disk.drbd2266.out‚0 network.sent‚ 1.7397477e+07 network.recv‚ 2.082526e+06 disk.sda3.out‚ 223.4140625 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2080.out‚0# disk.dm-0.out‚23506.21728515625 disk.sda2.in‚0 disk.drbd2265.out‚0 cpu‚25.361917362174655 disk.dm-1.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda2.out‚0( proc.memory.availableMB‚ 1425.4296875* proc.memory.percent‚16.130828857421875 disk.drbd1716.out‚0$ memory_percent‚91.29986763000488 disk.dm-1.out‚ 223.4140625( disk.nvme0c0n1.out‚23506.21728515625 disk.sda3.in‚ 0.01171875 proc.memory.rssMB‚2642.875 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 223.4140625 disk.rbd1.in‚0 disk.rbd1.out‚ 31.5205078125 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0* disk./.usagePercent‚20.276785412928273% disk./.usageGB‚302.13875579833984 disk.drbd2080.in‚0 disk.sda.in‚ 0.01171875 disk.rbd0.in‚0& disk.nvme0n1.out‚23506.21728515625 disk.drbd1716.in‚0Øje proc.cpu.threads‚17mE²àû‚(:’ ¯æÔ® îü gpu.0.gpu‚4.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚183.593' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.718599999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ïÿhü‚(:Ý ¯æÔ×óëÿ disk.dm-1.out‚ 223.8515625 cpu‚25.361117727864396 disk.rbd0.out‚0$ disk.nvme0n1.out‚23564.935546875 disk.dm-1.in‚ 0.01171875 disk.rbd1.in‚0 disk.rbd1.out‚ 31.6064453125& disk.nvme0c0n1.out‚23564.935546875 disk.sda3.out‚ 223.8515625! disk.dm-0.out‚23564.935546875 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.drbd1716.out‚0$ disk./.usageGB‚302.1389808654785 disk.nvme0n1.in‚0.078125* disk./.usagePercent‚20.276800517372795 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 proc.memory.rssMB‚2642.875 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0 disk.rbd0.in‚0* proc.memory.percent‚16.130828857421875 disk.sda2.in‚0 network.sent‚ 1.7519925e+07 disk.sda.out‚ 223.8515625 disk.sda1.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0# memory_percent‚91.2984848022461& proc.memory.availableMB‚ 1425.65625 network.recv‚ 2.087314e+06 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 proc.cpu.threads‚17Š=D32ý- _wandbruntime‚5817  _runtime‚5817¯ô+ÇþÁ _runtime‚5817.812735008 _step‚598 & lr-AdamW/pg1‚5.153394712882288e-05 & lr-AdamW/pg2‚5.153394712882288e-05 trainer/global_step‚7299 % _timestamp‚1.7884507023411236e+09ÉÙkÌÿÆ & lr-AdamW/pg1‚5.153394712882288e-05 & lr-AdamW/pg2‚5.153394712882288e-05 trainer/global_step‚7299 % _timestamp‚1.7884507023411236e+09 _runtime‚5817.812735008 _step‚598Öåv2€- _wandbruntime‚5817  _runtime‚5817fÃÕå'ßO UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 106409066 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚428 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚62090614 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 106409064 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚425315828959744 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚5817.813669371 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚425315828959744 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6121796554752 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚426 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚420256803502404 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚420256132025288 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚7299 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚97118303 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚431438299879936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚431437625514496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6422821193728 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 106409064 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚425315846962688 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚425315846962688 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 137969784 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚414156894602240 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6121796554752 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 137969784 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360¡£ôCê'‚äO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚425315828959744 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6121796554752 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚426 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚420256803502404 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚420256132025288 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚7299 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚97118303 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚431438299879936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚431437625514496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6422821193728 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 106409064 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚425315846962688 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚425315846962688 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 137969784 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚414156894602240 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6121796554752 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 137969784 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 106409066 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚428 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚62090614 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 106409064 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚425315828959744 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚5817.813669371 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680×Yÿ¾Ð2ƒ- _wandbruntime‚5818  _runtime‚5818‹*èï&„éM UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 244431384 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6122615982592 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 106423082 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚425372200001536 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚425372181998592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚97130960 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚97131380 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6100725335336 _step‚600 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 106423080 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚5818.505555876 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚425372181998592 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6122615982592 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚420311852590812 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6123272358400 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 106423082 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 138008302 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 137988238 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚470873406879232 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚420312524068136 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 trainer/global_step‚7299 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚62098782 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 172870656 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 148930560 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚425372200001536 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 244431384 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 137988238 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6423688618496 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚431495472359936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 106423080 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚35032180 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚414211798732800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚431495472359936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0þÅ—©…îM EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚425372181998592 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6122615982592 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚420311852590812 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6123272358400 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 106423082 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 138008302 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 137988238 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚470873406879232 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚420312524068136 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 trainer/global_step‚7299 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚62098782 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 172870656 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 148930560 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚425372200001536 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 244431384 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 137988238 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6423688618496 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚431495472359936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 106423080 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚35032180 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚414211798732800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚431495472359936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 244431384 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6122615982592 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 106423082 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚425372200001536 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚425372181998592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚97130960 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚97131380 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6100725335336 _step‚600 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 106423080 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚5818.505555876 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430ØÅf2†- _wandbruntime‚5818  _runtime‚5818út]‡× 1loss/SRqqt4/detection_loss‚1.6722456711743559 0loss/FBt1/assignment_loss‚0.5428952448960587 /loss/FRt2/detection_loss‚2.3362485331793628 $ _timestamp‚1.788450703035192e+09 1loss/SRqqt3/detection_loss‚1.1056835701068242 2loss/SRqqt4/assignment_loss‚1.4309137971805674 1loss/SRqqt2/detection_loss‚2.2261024937033653 2loss/SRqqt1/assignment_loss‚1.8040028118035372 2loss/SRqqt3/assignment_loss‚0.9899561136230154 .loss/FRt1/detection_loss‚3.012969589659146 .loss/FBt2/detection_loss‚2.064349311239579 /loss/FBt3/detection_loss‚0.8857223365691976 0loss/FRt1/assignment_loss‚3.3303380523409163 0loss/FRt4/assignment_loss‚2.8122173815357443 2loss/SRqqt2/assignment_loss‚1.5229959780056226 1loss/SRqqt1/detection_loss‚2.5137511158690735 .loss/FBt4/detection_loss‚3.367775099618094 _step‚601 0loss/FRt2/assignment_loss‚3.8169075863228903 1loss/FBt2/assignment_loss‚0.39680769877945216 /loss/FRt4/detection_loss‚2.3275200157445304 /loss/FBt1/detection_loss‚2.4624508284032345 epoch‚6 trainer/global_step‚7299 0loss/FRt3/assignment_loss‚1.8377412054687738 /loss/FRt3/detection_loss‚1.6867882849182934 %loss/total_loss‚44.78171395483362 _runtime‚ 5818.50668672 1loss/FBt3/assignment_loss‚0.37695048157168826 1loss/FBt4/assignment_loss‚0.25838075312029624¯¥ÓØbˆÜ 1loss/SRqqt1/detection_loss‚2.5137511158690735 .loss/FBt4/detection_loss‚3.367775099618094 _step‚601 0loss/FRt2/assignment_loss‚3.8169075863228903 1loss/FBt2/assignment_loss‚0.39680769877945216 /loss/FRt4/detection_loss‚2.3275200157445304 /loss/FBt1/detection_loss‚2.4624508284032345 epoch‚6 trainer/global_step‚7299 0loss/FRt3/assignment_loss‚1.8377412054687738 /loss/FRt3/detection_loss‚1.6867882849182934 %loss/total_loss‚44.78171395483362 _runtime‚ 5818.50668672 1loss/FBt3/assignment_loss‚0.37695048157168826 1loss/FBt4/assignment_loss‚0.25838075312029624 1loss/SRqqt4/detection_loss‚1.6722456711743559 0loss/FBt1/assignment_loss‚0.5428952448960587 /loss/FRt2/detection_loss‚2.3362485331793628 $ _timestamp‚1.788450703035192e+09 1loss/SRqqt3/detection_loss‚1.1056835701068242 2loss/SRqqt4/assignment_loss‚1.4309137971805674 1loss/SRqqt2/detection_loss‚2.2261024937033653 2loss/SRqqt1/assignment_loss‚1.8040028118035372 2loss/SRqqt3/assignment_loss‚0.9899561136230154 .loss/FRt1/detection_loss‚3.012969589659146 .loss/FBt2/detection_loss‚2.064349311239579 /loss/FBt3/detection_loss‚0.8857223365691976 0loss/FRt1/assignment_loss‚3.3303380523409163 0loss/FRt4/assignment_loss‚2.8122173815357443 2loss/SRqqt2/assignment_loss‚1.5229959780056226Ù9xn“‰‚(:ˆ ¯æÔáºý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚183.509' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.7018 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0kQiŠ‚(:Þ ¯æÔ»ˆðÿ disk.drbd1716.in‚0 network.sent‚ 1.7526493e+07 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda1.out‚0 disk.dm-1.out‚ 224.47265625 disk.drbd2080.out‚0 disk.sda3.out‚ 224.47265625 network.recv‚ 2.092335e+06 disk.rbd1.in‚0$ disk.nvme0n1.out‚24081.369140625 disk.nvme0c0n1.in‚0.078125* proc.memory.percent‚16.130828857421875 cpu‚25.360319585214405 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.drbd2265.out‚0 proc.memory.rssMB‚2642.875 disk.drbd1716.out‚0& disk.nvme0c0n1.out‚24081.369140625) disk./.usagePercent‚20.27769449568236$ disk./.usageGB‚302.1523017883301! disk.dm-0.out‚24081.369140625 disk.nvme0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 224.47265625 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.drbd2266.in‚0$ proc.memory.availableMB‚1425.625 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.rbd0.out‚0 disk.sda2.out‚0 disk.rbd1.out‚ 31.6689453125 disk.drbd2266.out‚0$ memory_percent‚91.29867553710938ÍÓgÄ“‹‚(:ˆ Ÿ¯æÔÞãúü gpu.0.gpu‚43.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.297' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.6594 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0|ÃÅgaŒ‚(:Ö Ÿ¯æÔ…Áãÿ disk.sda2.in‚0 disk.sda2.out‚0* proc.memory.percent‚16.130828857421875 disk.drbd2266.in‚0) disk./.usagePercent‚20.27417080798108% disk.nvme0n1.out‚24136.3916015625 disk.drbd1716.out‚0' disk.nvme0c0n1.out‚24136.3916015625 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 proc.memory.rssMB‚2642.875 network.recv‚ 2.097958e+06# disk./.usageGB‚302.099796295166" disk.dm-0.out‚24136.3916015625 disk.sda.in‚ 0.01171875 disk.sda.out‚225.125 disk.dm-1.out‚225.125 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.rbd0.in‚0 disk.sda1.in‚0) proc.memory.availableMB‚ 1425.62109375 disk.rbd1.in‚0 disk.rbd1.out‚ 31.7939453125 disk.nvme0n1.in‚0.078125 disk.sda3.out‚225.125 disk.sda1.out‚0 disk.dm-0.in‚0.078125$ memory_percent‚91.29869937896729 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 cpu‚25.359483967400536 disk.rbd0.out‚0 network.sent‚ 1.7634262e+07 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.011718754>蔞‚(:“ ®¯æÔæ·°ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚139.478' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.895600000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÛrKlŽ‚(:á ®¯æÔðèÊÿ network.sent‚ 1.7641091e+07 disk.sda3.out‚ 225.671875 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda.out‚ 225.671875 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 cpu‚25.358738633885373 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0$ memory_percent‚91.29722118377686 proc.memory.rssMB‚2642.875 disk.drbd2080.out‚0% disk.nvme0n1.out‚24182.3291015625 disk.drbd1716.out‚0' disk.nvme0c0n1.out‚24182.3291015625 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 31.8564453125 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 225.671875 disk.drbd1716.in‚0) proc.memory.availableMB‚ 1425.86328125* disk./.usagePercent‚20.276825350103618$ disk./.usageGB‚302.1393508911133* proc.memory.percent‚16.130828857421875 network.recv‚ 2.103536e+06 disk.drbd2266.in‚0" disk.dm-0.out‚24182.3291015625 disk.sda2.out‚0·”¿2- _wandbruntime‚5854  _runtime‚5854pétÇÁ & lr-AdamW/pg2‚5.063681385138409e-05 trainer/global_step‚7349 % _timestamp‚1.7884507390004294e+09 _runtime‚5854.472023066 _step‚602 & lr-AdamW/pg1‚5.063681385138409e-05|ΓÌ‘Æ & lr-AdamW/pg1‚5.063681385138409e-05 & lr-AdamW/pg2‚5.063681385138409e-05 trainer/global_step‚7349 % _timestamp‚1.7884507390004294e+09 _runtime‚5854.472023066 _step‚602ÚH¼Lk2’- _wandbruntime‚5854  _runtime‚5854#Øä'“ÞO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚434296937628160 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚473936176690688 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 107109864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 138895836 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚5854.473037701 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6465753556992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 138915874 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚416902101130240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 138895836 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚62499014 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6163438723072 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚428133498905088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6162782360576 $ _timestamp‚1.788450739001134e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚35253787 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚428133498905088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚97752799 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚35254242 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚434296263262720 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚434296263262720 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 246025740 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚467470604406272 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚62499012 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚467470580466176 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 246005700 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚428133480902144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚416902084090880 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 107109864 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚434296937628160 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚97753256 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚7349 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6465596224512 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 107109866 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 107109866 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚423042173892696 _step‚603 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6163438723072 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872¢ö·”ãO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚35254242 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚434296263262720 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚434296263262720 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 246025740 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚467470604406272 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚62499012 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚467470580466176 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 246005700 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚428133480902144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚416902084090880 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 107109864 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚434296937628160 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚97753256 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚7349 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6465596224512 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 107109866 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 107109866 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚423042173892696 _step‚603 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes–s[j.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6163438723072 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚434296937628160 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚473936176690688 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 107109864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 138895836 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚5854.473037701 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6465753556992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 138915874 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚416902101130240 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 138895836 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚62499014 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6163438723072 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚428133498905088 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6162782360576 $ _timestamp‚1.788450739001134e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚35253787 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚428133498905088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚97752799 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0ÛcvFc2•- _wandbruntime‚5855  _runtime‚5855qÇï&–éM XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚473998696119808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚434353435729408 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884507396817932e+09 _runtime‚5855.153860054 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6141560674744 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚434354110108160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚97765838 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚35258212 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 138934354 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 246058236 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 138934354 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚423098565935544 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6466464915968 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚467532231203840 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚434354110108160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚428189851943936 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚97765392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚428189851943936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6163601788416 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 107123882 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6164258164224 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚467532255143936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚62507180 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚446 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚416956988221440 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048ñýŠ[ô&—îM ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚428189851943936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6163601788416 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 107123882 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6164258164224 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚467532255143936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚62507180 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚446 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚416956988221440 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚473998696119808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚434353435729408 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884507396817932e+09 _runtime‚5855.153860054 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6141560674744 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚434354110108160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚97765838 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚35258212 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 138934354 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 246058236 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 138934354 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚423098565935544 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6466464915968 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚467532231203840 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚434354110108160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚428189851943936 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚97765392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚425828215760104 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚437155575376384 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 247652592 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 107810666 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 139841926 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚477061730217472 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚430951150847488 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚35479830 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚5891.188348521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚430951150847488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚477061536361984 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚437155575376384 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6204424528896 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 107810664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6203768166400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚62907414 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚419647307658240 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6180925141224 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 139821888 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚98387244 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚62907412 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚437154901010944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6508419012608 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _step‚607 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚98386794 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚430951132844544 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6508588928000 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚437154901010944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 $ _timestamp‚1.788450775716531e+09Ê^é'¤ãO H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚35479830 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚5891.188348521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚430951150847488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚448 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚477061536361984 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚437155575376384 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6204424528896 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 107810664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6203768166400 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚62907414 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚419647307658240 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6180925141224 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 139821888 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚98387244 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚62907412 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚437154901010944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6508419012608 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 _step‚607 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚98386794 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚430951132844544 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6508588928000 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚437154901010944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 $ _timestamp‚1.788450775716531e+09 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚425828215760104 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚437155575376384 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 247652592 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 107810666 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 139841926 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚477061730217472 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚430951150847488 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088ß$yÉÃ2¥- _wandbruntime‚5891  _runtime‚5891sõUï&¦éM OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚35483931 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 247685088 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6182396014152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 247685088 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 247665022 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚425884607802952 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚419702211788800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚437212747856384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 107824682 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 139860406 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚431007503886336 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 139840342 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚477124038192640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6204587594240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚62915580 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚431007503886336 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚98399511 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚419702194749440 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 107824682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚451 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6509440007680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884507764126337e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚437212073477632 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 107824680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6205243970048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6204587594240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 247665022 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 139860406 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚425883936325628 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 139840342 _runtime‚5891.884661509 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6181741576188 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚437212747856384 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚431007485883392 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 247685088 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 247665022 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚425884607802952 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚419702211788800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚437212747856384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.B¥ÁÑDeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 107824682 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 139860406 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚431007503886336 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 139840342 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚477124038192640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6204587594240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚62915580 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚431007503886336 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚98399511 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚419702194749440 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 107824682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚451 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6509440007680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884507764126337e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚437212073477632 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 107824680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6205243970048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6204587594240 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 247665022 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 139860406 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚425883936325628 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 139840342 _runtime‚5891.884661509 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6181741576188 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚437212747856384 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚431007485883392 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414অ½2¨- _wandbruntime‚5891  _runtime‚5891Ä8*T©Î 1loss/SRqqt2/assignment_loss‚1.355185386525201 /loss/FRt3/detection_loss‚1.7253272244158913 1loss/SRqqt2/detection_loss‚1.8833039211375373 _step‚609 .loss/FBt4/detection_loss‚4.363616704940796 trainer/global_step‚7399 /loss/FRt2/assignment_loss‚3.302822669869975 2loss/SRqqt1/assignment_loss‚2.5289529651403426 1loss/SRqqt4/assignment_loss‚1.333488667184221 0loss/FBt4/assignment_loss‚0.7301330491900444 /loss/FBt2/detection_loss‚1.9227474629878998 /loss/FRt4/assignment_loss‚2.644864658177909 0loss/FBt1/assignment_loss‚0.7338537438237722 .loss/FRt1/detection_loss‚3.409077464115052 1loss/SRqqt3/detection_loss‚1.2180651932481736 %loss/total_loss‚47.39111254051452 1loss/FBt3/assignment_loss‚0.24860759440737848 /loss/FRt4/detection_loss‚2.0520414265535645 /loss/FRt2/detection_loss‚2.6144358943167485 $ _timestamp‚1.788450776414265e+09 2loss/SRqqt3/assignment_loss‚0.9675903275963806 0loss/SRqqt4/detection_loss‚1.660566758364439 /loss/FBt3/detection_loss‚0.5114298983159044 /loss/SRqqt1/detection_loss‚2.90206894159317 epoch‚6 0loss/FBt2/assignment_loss‚0.7795837268543456 .loss/FBt1/detection_loss‚2.727661609649658 _runtime‚5891.886085676 /loss/FRt1/assignment_loss‚4.278412921088083 0loss/FRt3/assignment_loss‚1.4972743310180365]26WYªÓ epoch‚6 0loss/FBt2/assignment_loss‚0.7795837268543456 .loss/FBt1/detection_loss‚2.727661609649658 _runtime‚5891.886085676 /loss/FRt1/assignment_loss‚4.278412921088083 0loss/FRt3/assignment_loss‚1.4972743310180365 1loss/SRqqt2/assignment_loss‚1.355185386525201 /loss/FRt3/detection_loss‚1.7253272244158913 1loss/SRqqt2/detection_loss‚1.8833039211375373 _step‚609 .loss/FBt4/detection_loss‚4.363616704940796 trainer/global_step‚7399 /loss/FRt2/assignment_loss‚3.302822669869975 2loss/SRqqt1/assignment_loss‚2.5289529651403426 1loss/SRqqt4/assignment_loss‚1.333488667184221 0loss/FBt4/assignment_loss‚0.7301330491900444 /loss/FBt2/detection_loss‚1.9227474629878998 /loss/FRt4/assignment_loss‚2.644864658177909 0loss/FBt1/assignment_loss‚0.7338537438237722 .loss/FRt1/detection_loss‚3.409077464115052 1loss/SRqqt3/detection_loss‚1.2180651932481736 %loss/total_loss‚47.39111254051452 1loss/FBt3/assignment_loss‚0.24860759440737848 /loss/FRt4/detection_loss‚2.0520414265535645 /loss/FRt2/detection_loss‚2.6144358943167485 $ _timestamp‚1.788450776414265e+09 2loss/SRqqt3/assignment_loss‚0.9675903275963806 0loss/SRqqt4/detection_loss‚1.660566758364439 /loss/FBt3/detection_loss‚0.5114298983159044 /loss/SRqqt1/detection_loss‚2.90206894159317á¿ö¨Ã“«‚(:ˆ Û¯æÔÛ‰ý gpu.0.gpu‚55.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.841' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.1682 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“*¹p¬‚(:å Û¯æÔÇÿ˜ÿ disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 227.10546875* proc.memory.percent‚16.130828857421875 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda2.out‚0) proc.memory.availableMB‚ 1424.19140625 network.sent‚ 1.7760376e+07$ disk./.usageGB‚302.1683158874512 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 227.10546875( disk.nvme0c0n1.out‚24345.16845703125 disk.dm-0.in‚0.078125 disk.sda1.in‚0 proc.cpu.threads‚17 disk.sda.in‚ 0.01171875 disk.sda1.out‚0# memory_percent‚91.3074254989624 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda2.in‚0# disk.dm-0.out‚24345.16845703125 network.recv‚ 2.117105e+06& disk.nvme0n1.out‚24345.16845703125 disk.sda3.out‚ 227.10546875 disk.rbd0.in‚0 disk.rbd1.in‚0 proc.memory.rssMB‚2642.875( disk./.usagePercent‚20.2787692153111 disk.rbd1.out‚ 32.107421875 cpu‚25.35631255765296ÍÞ û“­‚(:ˆ ê¯æÔòÿ›ý gpu.0.gpu‚30.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.206' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.6412 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‘‚Ër®‚(:ç ê¯æÔ¹Õ¸€& disk.nvme0n1.out‚24391.09814453125 disk.nvme0c0n1.in‚0.078125 network.recv‚ 2.12436e+06 disk.nvme0n1.in‚0.078125 disk.sda3.out‚ 227.79296875 disk.rbd0.out‚0# disk.dm-0.out‚24391.09814453125 disk.sda.out‚ 227.79296875 disk.sda1.out‚0* proc.memory.percent‚16.130828857421875 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.sda3.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 proc.memory.rssMB‚2642.875 cpu‚25.355570052432398% disk./.usageGB‚302.18697357177734 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda2.in‚0 network.sent‚ 1.7869727e+07) disk./.usagePercent‚20.28002134816107 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.rbd1.out‚ 32.169921875 disk.drbd2080.in‚0 disk.drbd2266.in‚0$ memory_percent‚91.30797386169434( proc.memory.availableMB‚ 1424.1015625 proc.cpu.threads‚17 disk.dm-1.out‚ 227.79296875 disk.drbd2265.out‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚24391.09814453125 disk.sda1.in‚0V‘²Ö“¯‚(:ˆ ù¯æÔ¿°¬ý gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.274' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8548 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Há³*t°‚(:é ù¯æÔˆìœÿ disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 228.14453125 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0$ memory_percent‚91.30795001983643$ disk./.usageGB‚302.1974868774414 disk.rbd0.out‚0 disk.drbd1716.in‚0 disk.sda3.out‚ 228.14453125& disk.nvme0n1.out‚24448.14990234375( disk.nvme0c0n1.out‚24448.14990234375 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.out‚0) proc.memory.availableMB‚ 1424.10546875 disk.drbd2080.in‚0 proc.memory.rssMB‚2642.875 cpu‚25.354768937192905 network.sent‚ 1.7874975e+07 network.recv‚ 2.127478e+06 disk.sda3.in‚ 0.01171875 disk.sda2.out‚0 disk.rbd1.out‚ 32.169921875 disk.drbd2265.in‚0* proc.memory.percent‚16.130828857421875* disk./.usagePercent‚20.280726904925437 disk.drbd2266.out‚0# disk.dm-0.out‚24448.14990234375 disk.sda.out‚ 228.14453125 disk.sda1.in‚0 disk.rbd1.in‚0/!2±- _wandbruntime‚5928  _runtime‚5928ñVÏ Ç²Á _step‚610 & lr-AdamW/pg1‚4.885067746031729e-05 & lr-AdamW/pg2‚4.885067746031729e-05 trainer/global_step‚7449 % _timestamp‚1.7884508128534648e+09 _runtime‚5928.325108035”Öh×Ì³Æ _runtime‚5928.325108035 _step‚610 & lr-AdamW/pg1‚4.885067746031729e-05 & lr-AdamW/pg2‚4.885067746031729e-05 trainer/global_step‚7449 % _timestamp‚1.7884508128534648e+09ⳃ¼þ2´- _wandbruntime‚5928  _runtime‚5928Ö¯­+å'µßO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚35707887 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚422392514186240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6222414918388 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚440014213124608 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚433768784786944 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 108511466 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6551516053504 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6244753972224 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚480187194226176 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚473635654232576 _step‚611 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚63315814 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚35708326 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚99023699 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6244753972224 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚433768802789888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 108511464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 140747940 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚440013538759168 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚63315812 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 140747940 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚433768784786944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚440013538759168 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6245410334720 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚480187006662144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6221760480632 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚433768802789888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚99024140 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚428614257627512 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚422392497146880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚7449 _runtime‚5928.325968219 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 108511464 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚428614929104628 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚440014213124608 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 140767978 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚473635678172672 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6245410334720 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 249279444 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844òH#2õ¶äO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚440014213124608 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 140767978 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚473635678172672 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6245410334720 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 249279444 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚35707887 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚422392514186240 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6222414918388 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚440014213124608 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚433768784786944 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 108511466 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6551516053504 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6244753972224 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚480187194226176 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚473635654232576 _step‚611 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚63315814 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚35708326 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚99023699 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6244753972224 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚433768802789888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 108511464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 140747940 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚440013538759168 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚63315812 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 140747940 YEDeviceStatz³Œõ sMonitor.on_train_batch_start/active_bytes.large_pool.freed‚433768784786944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚440013538759168 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6245410334720 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚480187006662144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6221760480632 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚433768802789888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚99024140 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚428614257627512 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚422392497146880 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚7449 _runtime‚5928.325968219 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 108511464 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚428614929104628 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0ãÿE¦2·- _wandbruntime‚5929  _runtime‚5929ÎDüßï&¸éM K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚99036289 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚428670649670360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚422447418316800 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 249311940 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚433825155828736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 249311940 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 108525480 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 140786458 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚63323980 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚422447401277440 _runtime‚5929.015843502 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚612 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚473697328910336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 108525482 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 % _timestamp‚1.7884508135436347e+09 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚473697304970240 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚99036729 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚35712309 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚480249702154752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6552373244416 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 249291874 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6245573400064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚480249512506880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 trainer/global_step‚7449 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414uÓJô&¹îM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚422447401277440 _runtime‚5929.015843502 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚612 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚473697328910336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 108525482 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 % _timestamp‚1.7884508135436347e+09 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚473697304970240 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚99036729 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚35712309 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚480249702154752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6552373244416 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 249291874 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6245573400064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚480249512506880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 trainer/global_step‚7449 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚99036289 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚428670649670360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚422447418316800 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 249311940 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚433825155828736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 249311940 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 108525480 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 140786458 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚63323980ä• Ø2º- _wandbruntime‚5929  _runtime‚5929ï |·Z»Ô 1loss/SRqqt4/assignment_loss‚1.403000427240675 0loss/FBt1/assignment_loss‚0.5558107474016164 % _timestamp‚1.7884508135457873e+09 _runtime‚ 5929.01727876 0loss/FRt4/assignment_loss‚2.9822264422069895 0loss/FBt2/assignment_loss‚0.6625323364617084 .loss/FRt2/detection_loss‚2.170069120031722 1loss/SRqqt3/detection_loss‚1.2246962149034848 /loss/FBt2/detection_loss‚2.3345088586211205 0loss/FRt2/assignment_loss‚2.3295404467176883 &loss/total_loss‚45.775666674379416 epoch‚6 2loss/SRqqt1/assignment_loss‚2.5193944115329674 trainer/global_step‚7449 0loss/FRt3/assignment_loss‚1.5393512343818492 .loss/FRt1/detection_loss‚3.293754380000265 /loss/FRt1/assignment_loss‚4.448556727484653 2loss/SRqqt2/assignment_loss‚1.5830390551903595 2loss/SRqqt3/assignment_loss‚1.1138126185925847 /loss/FRt3/detection_loss‚1.8273794686591083 .loss/FRt4/detection_loss‚2.074320707537911 1loss/SRqqt4/detection_loss‚1.5532546250657602 /loss/FBt1/detection_loss‚2.9190491331475124 /loss/FBt3/detection_loss‚0.8970800704012315 1loss/FBt3/assignment_loss‚0.30351555014492854 0loss/FBt4/assignment_loss‚0.5576225698089481 1loss/SRqqt1/detection_loss‚2.6578609259040267 /loss/FBt4/detection_loss‚2.8484280109405518 _step‚613 0loss/SRqqt2/detection_loss‚1.976862592001756žÓfe_¼Ù 1loss/SRqqt4/assignment_loss‚1.403000427240675 0loss/FBt1/assignment_loss‚0.5558107474016164 % _timestamp‚1.7884508135457873e+09 _runtime‚ 5929.01727876 0loss/FRt4/assignment_loss‚2.9822264422069895 0loss/FBt2/assignment_loss‚0.6625323364617084 .loss/FRt2/detection_loss‚2.170069120031722 1loss/SRqqt3/detection_loss‚1.2246962149034848 /loss/FBt2/detection_loss‚2.3345088586211205 0loss/FRt2/assignment_loss‚2.3295404467176883 &loss/total_loss‚45.775666674379416 epoch‚6 2loss/SRqqt1/assignment_loss‚2.5193944115329674 trainer/global_step‚7449 0loss/FRt3/assignment_loss‚1.5393512343818492 .loss/FRt1/detection_loss‚3.293754380000265 /loss/FRt1/assignment_loss‚4.448556727484653 2loss/SRqqt2/assignment_loss‚1.5830390551903595 2loss/SRqqt3/assignment_loss‚1.1138126185925847 /loss/FRt3/detection_loss‚1.8273794686591083 .loss/FRt4/detection_loss‚2.074320707537911 1loss/SRqqt4/detection_loss‚1.5532546250657602 /loss/FBt1/detection_loss‚2.9190491331475124 /loss/FBt3/detection_loss‚0.8970800704012315 1loss/FBt3/assignment_loss‚0.30351555014492854 0loss/FBt4/assignment_loss‚0.5576225698089481 1loss/SRqqt1/detection_loss‚2.6578609259040267 /loss/FBt4/detection_loss‚2.8484280109405518 _step‚613 0loss/SRqqt2/detection_loss‚1.976862592001756åzÓÑ?’½‚(:‡ ˆ°æÔ“©µþ gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚174.945' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.989 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0V}×v¾‚(:ë ˆ°æÔˆ¢Ïÿ# disk.dm-0.out‚24480.08740234375 disk.sda.out‚ 228.75390625 network.recv‚ 2.132291e+06 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚24480.08740234375 disk.drbd1716.in‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0 disk.sda2.out‚0 cpu‚25.353999593344557 disk.dm-1.out‚ 228.75390625 disk.drbd2080.in‚0 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.sda2.in‚0 proc.cpu.threads‚17* disk./.usagePercent‚20.281819033066505 disk.drbd2265.out‚0 disk.sda3.out‚ 228.75390625 disk.dm-0.in‚0.078125* proc.memory.percent‚16.130828857421875 disk.rbd0.out‚0% disk./.usageGB‚302.21376037597656 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.rbd1.out‚ 32.2646484375 network.sent‚ 1.7881402e+07( disk.nvme0c0n1.out‚24480.08740234375) proc.memory.availableMB‚ 1423.82421875 disk.sda1.out‚0$ memory_percent‚91.30966663360596 disk.drbd2266.in‚0 disk.rbd1.in‚0 proc.memory.rssMB‚2642.875ˆ$åž¿‚(:“ —°æÔó¥éü gpu.0.gpu‚55.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.834' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.766799999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0h #½qÀ‚(:æ —°æÔˆ¾éÿ disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.drbd2265.in‚0 proc.memory.rssMB‚2642.875 disk.drbd2266.in‚0 disk.sda.out‚ 229.23828125 disk.sda1.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 32.3896484375$ memory_percent‚91.30816459655762( proc.memory.availableMB‚ 1424.0703125 disk.rbd1.in‚0 disk.dm-1.out‚ 229.23828125 disk.nvme0n1.in‚0.078125 network.recv‚ 2.137834e+06% disk./.usageGB‚302.22455978393555 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0) disk./.usagePercent‚20.28254379039594 disk.sda3.out‚ 229.23828125 disk.dm-0.in‚0.078125" disk.dm-0.out‚24552.3154296875 disk.sda2.in‚0 disk.drbd2265.out‚0* proc.memory.percent‚16.130828857421875 cpu‚25.353235896207096 disk.sda3.in‚ 0.01171875 disk.drbd1716.in‚0 network.sent‚ 1.7989304e+07' disk.nvme0c0n1.out‚24552.3154296875% disk.nvme0n1.out‚24552.3154296875 disk.drbd1716.out‚0 proc.cpu.threads‚17 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0¡³ßð2Á- _wandbruntime‚5964  _runtime‚5964I6slÇÂÁ _step‚614 & lr-AdamW/pg1‚4.796204657483606e-05 & lr-AdamW/pg2‚4.796204657483606e-05 trainer/global_step‚7499 % _timestamp‚1.7884508490799918e+09 _runtime‚5964.551588411ü/PØÌÃÆ & lr-AdamW/pg1‚4.796204657483606e-05 & lr-AdamW/pg2‚4.796204657483606e-05 trainer/global_step‚7499 % _timestamp‚1.7884508490799918e+09 _runtime‚5964.551588411 _step‚614æ=*t/2Ä- _wandbruntime‚5964  _runtime‚5964Vø^ÅÞO ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚436586454732288 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 109212264 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚35936300 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 141673992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚442872850872832 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 250906296 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚476718215055872 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6286396140544 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚7499 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 250886256 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚63724214 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚35935857 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6285739778048 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.788450849080718e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚442872850872832 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start.ÐdšW/inactive_split.all.peak‚908 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚425137703674880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 109212264 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6594305062912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 141694030 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚431400299494920 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚436586436729344 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚436586454732288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚63724212 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚99660069 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 109212266 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 _step‚615 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚442872176507392 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 141673992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6594472881152 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6286396140544 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 trainer/global_step‚7499 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 250886256 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚63724214 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚35935857 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6285739778048 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.788450849080718e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚442872850872832 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚425137703674880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 109212264 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚445 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6594305062912 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 141694030 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚431400299494920 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚436586436729344 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚436586454732288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚63724212 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚99660069 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 109212266 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 _step‚615 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚442872176507392 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 141673992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6594472881152 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 141692446 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚431456020060444 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6595327329792 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6286559205888 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚5965.260627022 _step‚616 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6595159524864 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 250938792 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚483375193123328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚63732380 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚476779865793536 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚483375001378304 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 109226280 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚436642807771136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚436642807771136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 109226280 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 109226282 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6286559205888 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚442929348974080 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6287215581696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚425192624844800 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚99672778 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚442930023352832 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928~&&|ÉîM XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚483375001378304 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 109226280 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚436642807771136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚436642807771136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 109226280 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 109226282 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6286559205888 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚442929348974080 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6287215581696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚425192624844800 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚99672778 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚442930023352832 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 trainer/global_step‚7499 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚35940845 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6287215581696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6263412255004 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚436642789768192 ICDeviceStatsMonitor.on_train_batch_end/allocationþ” 1x.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 141692446 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 % _timestamp‚1.7884508497885087e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 141712510 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 141692446 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚431456020060444 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6595327329792 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6286559205888 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _runtime‚5965.260627022 _step‚616 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6595159524864 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 250938792 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚483375193123328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚63732380 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚476779865793536 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430è¢Á 2Ê- _wandbruntime‚5965  _runtime‚5965QJÈüYËÓ _step‚617 /loss/FRt3/detection_loss‚1.8285078054987918 1loss/SRqqt3/assignment_loss‚1.031545929494314 2loss/SRqqt4/assignment_loss‚1.3904981907043192 0loss/FBt2/assignment_loss‚0.6180767461967959 trainer/global_step‚7499 /loss/FRt2/assignment_loss‚3.130243275525435 epoch‚6 /loss/FRt1/assignment_loss‚4.581822893836281 2loss/SRqqt1/assignment_loss‚2.2290356873438277 /loss/FRt4/detection_loss‚2.1919621124863626 1loss/SRqqt1/detection_loss‚2.7300159807862907 1loss/SRqqt4/detection_loss‚1.7808993923719283 %loss/total_loss‚47.30061114367669 1loss/SRqqt3/detection_loss‚1.1988900388241746 /loss/FBt2/detection_loss‚2.5592384239037833 /loss/FBt4/detection_loss‚3.6577136516571045 1loss/FBt1/assignment_loss‚0.42767412731263577 /loss/FRt1/detection_loss‚3.2287893999706614 .loss/FBt1/detection_loss‚2.614278014217104 /loss/FBt3/detection_loss‚0.8371798941201862 % _timestamp‚1.7884508497902944e+09 _runtime‚ 5965.26178683 0loss/FBt3/assignment_loss‚0.2431000288721118 /loss/FRt2/detection_loss‚2.2285633002812006 0loss/FRt3/assignment_loss‚1.8206468481643527 /loss/FRt4/assignment_loss‚3.218908053636551 1loss/SRqqt2/assignment_loss‚1.451570224897428 0loss/FBt4/assignment_loss‚0.3570758285862515 1loss/SRqqt2/detection_loss‚1.9443752949888056ÝZGB^ÌØ 1loss/SRqqt2/assignment_loss‚1.451570224897428 0loss/FBt4/assignment_loss‚0.3570758285862515 1loss/SRqqt2/detection_loss‚1.9443752949888056 _step‚617 /loss/FRt3/detection_loss‚1.8285078054987918 1loss/SRqqt3/assignment_loss‚1.031545929494314 2loss/SRqqt4/assignment_loss‚1.3904981907043192 0loss/FBt2/assignment_loss‚0.6180767461967959 trainer/global_step‚7499 /loss/FRt2/assignment_loss‚3.130243275525435 epoch‚6 /loss/FRt1/assignment_loss‚4.581822893836281 2loss/SRqqt1/assignment_loss‚2.2290356873438277 /loss/FRt4/detection_loss‚2.1919621124863626 1loss/SRqqt1/detection_loss‚2.7300159807862907 1loss/SRqqt4/detection_loss‚1.7808993923719283 %loss/total_loss‚47.30061114367669 1loss/SRqqt3/detection_loss‚1.1988900388241746 /loss/FBt2/detection_loss‚2.5592384239037833 /loss/FBt4/detection_loss‚3.6577136516571045 1loss/FBt1/assignment_loss‚0.42767412731263577 /loss/FRt1/detection_loss‚3.2287893999706614 .loss/FBt1/detection_loss‚2.614278014217104 /loss/FBt3/detection_loss‚0.8371798941201862 % _timestamp‚1.7884508497902944e+09 _runtime‚ 5965.26178683 0loss/FBt3/assignment_loss‚0.2431000288721118 /loss/FRt2/detection_loss‚2.2285633002812006 0loss/FRt3/assignment_loss‚1.8206468481643527 /loss/FRt4/assignment_loss‚3.218908053636551é©G¥·‘Í‚(:† ¦°æÔÛ˧ý gpu.0.gpu‚36.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚173.51' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.702 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 CšBh΂(:Ý ¦°æԫׇ€ disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚24592.5771484375 disk.sda2.in‚0$ memory_percent‚91.30878448486328 cpu‚25.35247559664103 disk.sda3.out‚ 229.953125 disk.sda.in‚ 0.01171875& proc.memory.availableMB‚ 1423.96875 disk.rbd1.in‚0* disk./.usagePercent‚20.281203590954192 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875" disk./.usageGB‚302.20458984375 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.drbd1716.out‚0 proc.cpu.threads‚17" disk.dm-0.out‚24592.5771484375 disk.sda.out‚ 229.94921875 disk.dm-1.out‚ 229.953125 network.recv‚ 2.142791e+06 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.rbd1.out‚ 32.4521484375 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚24592.5771484375 disk.drbd2080.out‚0 disk.drbd2266.out‚0 network.sent‚ 1.7996005e+07 disk.drbd2080.in‚0 disk.drbd1716.in‚0 proc.memory.rssMB‚2642.875* proc.memory.percent‚16.130828857421875!~˞ς(:“ µ°æÔÏ„ý gpu.0.gpu‚45.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚144.992' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.998399999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Âhû˜hЂ(:Ý µ°æÔ¢Êó disk.sda1.out‚0 disk.rbd1.out‚ 32.4521484375 cpu‚25.351663381130344 proc.cpu.threads‚17 disk.drbd1716.in‚0 disk.drbd1716.out‚0 network.recv‚ 2.148295e+06* disk./.usagePercent‚20.282559150847995$ disk./.usageGB‚302.2247886657715 disk.drbd2266.out‚0 disk.sda.out‚230.4375 disk.sda2.in‚0 disk.sda2.out‚0 disk.sda3.in‚ 0.01171875# disk.dm-0.out‚24643.71826171875 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚24643.71826171875 disk.dm-1.in‚ 0.01171875 proc.memory.rssMB‚2642.875 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.dm-1.out‚230.4375( proc.memory.availableMB‚ 949.44921875 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0$ memory_percent‚94.20502185821533* proc.memory.percent‚16.130828857421875 disk.rbd0.out‚0 disk.drbd2265.in‚0 network.sent‚ 1.8103775e+07 disk.dm-0.in‚0.078125 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚24643.71826171875 disk.sda3.out‚230.4375 disk.sda.in‚ 0.01171875° ²ê“Ñ‚(:ˆ Ä°æÔÐó„ý gpu.0.gpu‚41.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.262' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.8524 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ˆËÌrkÒ‚(:à Ä°æÔžŒ„ÿ network.sent‚ 1.8110331e+07 disk.sda2.out‚0 disk.drbd2080.in‚0$ memory_percent‚94.20504570007324 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 proc.cpu.threads‚17 network.recv‚ 2.153173e+06 disk.sda.in‚ 0.01171875 disk.sda1.in‚0' proc.memory.availableMB‚ 949.4453125 proc.memory.rssMB‚2642.875) disk./.usagePercent‚20.28090073404119% disk./.usageGB‚302.20007705688477 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚24690.9384765625 disk.rbd0.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2266.out‚0" disk.dm-0.out‚24690.9384765625 disk.sda.out‚ 231.09765625* proc.memory.percent‚16.130828857421875 disk.sda2.in‚0 disk.rbd1.out‚ 32.484375% disk.nvme0n1.out‚24690.9384765625 disk.dm-1.out‚ 231.09765625 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 cpu‚25.35093800195011 disk.sda3.out‚ 231.09765625÷^Ì“2Ó- _wandbruntime‚6001  _runtime‚6001ðWÇÔÁ trainer/global_step‚7549 % _timestamp‚1.7884508860500767e+09 _runtime‚6001.521663131 _step‚618 & lr-AdamW/pg1‚4.707662102498041e-05 & lr-AdamW/pg2‚4.707662102498041e-05rbÕÛÌÕÆ & lr-AdamW/pg1‚4.707662102498041e-05 & lr-AdamW/pg2‚4.707662102498041e-05 trainer/global_step‚7549 % _timestamp‚1.7884508860500767e+09 _runtime‚6001.521663131 _step‚618êkÇgL2Ö- _wandbruntime‚6001  _runtime‚6001óÌÓç'×áO QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6326725583872 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6304085597204 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 109913066 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚36165325 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚434186341362328 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 trainer/global_step‚7549 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚445731488621056 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 142620082 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚486438163457536 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚427882927242240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚439404088671744 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚36164872 _step‚619 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 252513108 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚445730814255616 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚439404106674688 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 100297939 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚427882910202880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 252533148 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚439404106674688 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚486438344730112 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7884508860509062e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 109913064 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 100297484 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6303431159448 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6327381946368 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚6001.522748199 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚434187012839444 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6637435458560 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚64132614 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚64132612 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚445730814255616 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚439404106674688 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 100297939 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚427882910202880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 252533148 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚439404106674688 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚486438344730112 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7884508860509062e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 109913064 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 100297484 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6303431159448 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6327381946368 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _runtime‚6001.522748199 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚434187012839444 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6637435458560ëþwJÁ2Ù- _wandbruntime‚6002  _runtime‚6002j”¨(ÚéM OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 252565644 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚7549 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚445788661101056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6638306517504 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 109927080 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 142638562 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6304247594412 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚427937814333440 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚36169396 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6304902032376 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 142618498 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚445787986722304 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚427937831372800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 252565644 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 100310176 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚434242733405176 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 109927082 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6328201387520 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 142618498 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 252565644 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 100310176 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚434242733405176 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 109927082 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚440 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6328201387520 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 142618498 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚7549 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚445788661101056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6638306517504 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 109927080 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 142638562 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6304247594412 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚427937814333440 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚36169396 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6304902032376 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 142618498 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚445787986722304 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚427937831372800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0ìbîá2Ü- _wandbruntime‚6002  _runtime‚6002Ž©-ÉXÝÒ /loss/FRt1/assignment_loss‚4.150321499161098 1loss/SRqqt1/assignment_loss‚1.836928328982106 epoch‚6 0loss/FRt3/assignment_loss‚1.7250163119572859 2loss/SRqqt4/assignment_loss‚1.6969075480411793 1loss/FBt3/assignment_loss‚0.20482288651034963 /loss/FRt4/detection_loss‚2.2650280183553697 1loss/SRqqt3/detection_loss‚1.0570860197146734 0loss/FBt2/assignment_loss‚0.4390074602242335 0loss/SRqqt1/detection_loss‚2.722670747174157 1loss/SRqqt2/detection_loss‚1.8480062580977876 0loss/FRt4/assignment_loss‚2.9652548742294313 $ _timestamp‚1.788450886755254e+09 /loss/FRt2/assignment_loss‚3.064814969187691 0loss/FBt1/assignment_loss‚0.4790899974857609 /loss/FRt3/detection_loss‚1.6907010546240668 /loss/FBt3/detection_loss‚0.6079090743139386 trainer/global_step‚7549 2loss/SRqqt2/assignment_loss‚1.5635736278879147 1loss/FBt4/assignment_loss‚0.45136580753670485 -loss/FRt1/detection_loss‚3.22624390280765 1loss/SRqqt4/detection_loss‚1.5285363662345657 -loss/FBt4/detection_loss‚2.22932196325726 /loss/FRt2/detection_loss‚2.1392360577980676 /loss/FBt1/detection_loss‚2.6414331793785095 /loss/FBt2/detection_loss‚2.7634702053937046 2loss/SRqqt3/assignment_loss‚1.1051326808674882 %loss/total_loss‚44.40187883922099 _runtime‚6002.226945933 _step‚621ˆûʇ]Þ× /loss/FRt1/assignment_loss‚4.150321499161098 1loss/SRqqt1/assignment_loss‚1.836928328982106 epoch‚6 0loss/FRt3/assignment_loss‚1.7250163119572859 2loss/SRqqt4/assignment_loss‚1.6969075480411793 1loss/FBt3/assignment_loss‚0.20482288651034963 /loss/FRt4/detection_loss‚2.2650280183553697 1loss/SRqqt3/detection_loss‚1.0570860197146734 0loss/FBt2/assignment_loss‚0.4390074602242335 0loss/SRqqt1/detection_loss‚2.722670747174157 1loss/SRqqt2/detection_loss‚1.8480062580977876 0loss/FRt4/assignment_loss‚2.9652548742294313 $ _timestamp‚1.788450886755254e+09 /loss/FRt2/assignment_loss‚3.064814969187691 0loss/FBt1/assignment_loss‚0.4790899974857609 /loss/FRt3/detection_loss‚1.6907010546240668 /loss/FBt3/detection_loss‚0.6079090743139386 trainer/global_step‚7549 2loss/SRqqt2/assignment_loss‚1.5635736278879147 1loss/FBt4/assignment_loss‚0.45136580753670485 -loss/FRt1/detection_loss‚3.22624390280765 1loss/SRqqt4/detection_loss‚1.5285363662345657 -loss/FBt4/detection_loss‚2.22932196325726 /loss/FRt2/detection_loss‚2.1392360577980676 /loss/FBt1/detection_loss‚2.6414331793785095 /loss/FBt2/detection_loss‚2.7634702053937046 2loss/SRqqt3/assignment_loss‚1.1051326808674882 %loss/total_loss‚44.40187883922099 _runtime‚6002.226945933 _step‚621íÁ5¦’ß‚(:‡ Ó°æÔýïÿü gpu.0.gpu‚1.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚124.084' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.8168 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‚^žhà‚(:Ý Ó°æÔ¼Óÿ disk.drbd2266.in‚0 disk.sda1.in‚0 disk.rbd1.out‚ 32.6953125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2266.out‚0" disk.dm-0.out‚24754.2119140625 proc.memory.rssMB‚2642.875* proc.memory.percent‚16.130828857421875 network.sent‚ 1.8115471e+07 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 231.484375 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚24754.2119140625' disk.nvme0c0n1.out‚24754.2119140625 disk.sda.out‚ 231.484375 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.dm-0.in‚0.078125 cpu‚25.350190517439664 network.recv‚ 2.156466e+06 disk.rbd0.out‚0 disk.sda1.out‚0$ disk./.usageGB‚302.2549018859863 disk.rbd1.in‚0 disk.dm-1.out‚ 231.484375 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875$ memory_percent‚94.20530796051025( proc.memory.availableMB‚ 949.40234375* disk./.usagePercent‚20.284580074323287 disk.sda2.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0 disk.drbd2080.in‚0ÌÜx“á‚(:ˆ â°æÔÓïý gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚126.081' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.2162 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0dBfHjâ‚(:ß â°æÔ³¯¶ÿ network.sent‚ 1.8224932e+07' disk.nvme0c0n1.out‚24796.9580078125 disk.dm-0.in‚0.078125 disk.sda2.out‚0$ memory_percent‚94.20530796051025* proc.memory.percent‚16.130828857421875) disk./.usagePercent‚20.28605749380339% disk./.usageGB‚302.27691650390625% disk.nvme0n1.out‚24796.9580078125 disk.sda1.in‚0 network.recv‚ 2.163827e+06 disk.drbd2080.in‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 232.5546875 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0" disk.dm-0.out‚24796.9580078125 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 33.0234375 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.dm-1.out‚ 232.5546875 disk.sda3.out‚ 232.5546875 cpu‚25.34940334627982 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0( proc.memory.availableMB‚ 949.40234375 proc.cpu.threads‚17 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda2.in‚0 proc.memory.rssMB‚2642.875A¸.02ã- _wandbruntime‚6038  _runtime‚6038Ú†^ÇäÁ trainer/global_step‚7599 % _timestamp‚1.7884509229779654e+09 _runtime‚6038.449530749 _step‚622 & lr-AdamW/pg1‚4.619458533210785e-05 & lr-AdamW/pg2‚4.619458533210785e-05ƒ…ÌåÆ & lr-AdamW/pg2‚4.619458533210785e-05 trainer/global_step‚7599 % _timestamp‚1.7884509229779654e+09 _runtime‚6038.449530749 _step‚622 & lr-AdamW/pg1‚4.619458533210785e-05îÝ!…ï2æ- _wandbruntime‚6038  _runtime‚6038LÌìçáO G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 110613864 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6367711389696 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚442221740614144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚36392991 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6344266498856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚448589452003840 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚442221758617088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚489563783449600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6344920936612 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 110613866 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6680494627328 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚6038.450464281 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6367711389696 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 100934467 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚489563595885568 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚482883288822272 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 A7DeviceStatsMonitor.on_train_batch_start/active.all.peaåòJ•Ù k‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 110613864 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 143546134 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚448590126369280 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 254160000 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚430628133770240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 143526096 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7884509229786386e+09 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 _step‚623 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚448589452003840 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚64541012 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚482883264882176 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚436972383229736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚448590126369280 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756®\ì'èæO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 110613864 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 143546134 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚448590126369280 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 254160000 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚430628133770240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 143526096 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7884509229786386e+09 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 _step‚623 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚448589452003840 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚64541012 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚482883264882176 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚436972383229736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚448590126369280 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 110613864 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6367711389696 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚442221740614144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚36392991 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6344266498856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚448589452003840 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚442221758617088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚489563783449600 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6344920936612 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 110613866 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6680494627328 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚6038.450464281 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6367711389696 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 100934467 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚489563595885568 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚482883288822272 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096ïè†hF2é- _wandbruntime‚6039  _runtime‚6039«ï&ð&êêM H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 254192496 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚448647298849280 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚430683020861440 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 143564614 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚461 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6369187193344 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 $ _timestamp‚1.788450923903552e+09 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚64549182 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 110627880 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚482944939559936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6368530817536 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚437028103795260 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6681360112640 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 110627882 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚442278111655936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 254172430 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚430683037900800 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 143544550 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚482944915619840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 254192496 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚6039.375632017 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚442278093652992 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚36397506 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚489626299672576 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 143544550 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6369187193344 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6345737371784 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚437028775272584 _step‚624 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6345082933820 trainer/global_step‚7599 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚448647298849280 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚459 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚64549180 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 110627880 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 143564614 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 100947147 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 100946686 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚36397965 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6368530817536 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2ÐËö"ëïM L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚64549180 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 110627880 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 143564614 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 100947147 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 100946686 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚36397965 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6368530817536 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 254192496 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚448647298849280 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚430683020861440 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 143564614 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚461 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6369187193344 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 $ _timestamp‚1.788450923903552e+09 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚64549182 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 110627880 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚482944939559936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6368530817536 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚437028103795260 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6681360112640 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 110627882 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚442278111655936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 254172430 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚430683037900800 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 143544550 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚482944915619840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 254192496 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚6039.375632017 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚442278093652992 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚36397506 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚489626299672576 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 143544550 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6369187193344 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6345737371784 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034W {ÿ944 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚437028775272584 _step‚624 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6345082933820 trainer/global_step‚7599 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚448647298849280 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚459 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964ðtÙ2ì- _wandbruntime‚6039  _runtime‚6039ôýªeZíÔ 1loss/SRqqt2/assignment_loss‚1.532826676787365 2loss/SRqqt3/assignment_loss‚1.0203677078835287 /loss/FRt4/detection_loss‚2.1706248156049033 0loss/SRqqt3/detection_loss‚1.288214198361009 2loss/SRqqt4/assignment_loss‚1.3685227925756147 .loss/FBt4/detection_loss‚2.489545399492437 /loss/FRt2/assignment_loss‚3.451342338323593 1loss/FBt2/assignment_loss‚0.46206453135335535 /loss/FRt3/detection_loss‚1.8588239978602592 1loss/SRqqt4/detection_loss‚1.4272765500204905 .loss/FBt1/detection_loss‚2.801566253105799 /loss/FBt3/detection_loss‚0.6468040167708837 0loss/FRt4/assignment_loss‚2.4232864022254943 0loss/FBt4/assignment_loss‚0.5192448388436126 1loss/SRqqt1/detection_loss‚2.5534532166296438 &loss/total_loss‚44.280450569157516 _step‚625 1loss/FBt1/assignment_loss‚0.25329812198920837 /loss/FRt1/detection_loss‚3.2990916339974654 /loss/FRt2/detection_loss‚2.2339853191375734 /loss/FBt2/detection_loss‚2.2251991822439083 trainer/global_step‚7599 _runtime‚6039.376708698 /loss/FRt1/assignment_loss‚4.154544817773919 1loss/SRqqt1/assignment_loss‚2.139225570905593 1loss/FBt3/assignment_loss‚0.14262694481229682 /loss/FRt3/assignment_loss‚1.735559935265399 0loss/SRqqt2/detection_loss‚2.082955307194165 epoch‚6 % _timestamp‚1.7884509239052145e+09ôô¼b_îÙ .loss/FBt1/detection_loss‚2.801566253105799 /loss/FBt3/detection_loss‚0.6468040167708837 0loss/FRt4/assignment_loss‚2.4232864022254943 0loss/FBt4/assignment_loss‚0.5192448388436126 1loss/SRqqt1/detection_loss‚2.5534532166296438 &loss/total_loss‚44.280450569157516 _step‚625 1loss/FBt1/assignment_loss‚0.25329812198920837 /loss/FRt1/detection_loss‚3.2990916339974654 /loss/FRt2/detection_loss‚2.2339853191375734 /loss/FBt2/detection_loss‚2.2251991822439083 trainer/global_step‚7599 _runtime‚6039.376708698 /loss/FRt1/assignment_loss‚4.154544817773919 1loss/SRqqt1/assignment_loss‚2.139225570905593 1loss/FBt3/assignment_loss‚0.14262694481229682 /loss/FRt3/assignment_loss‚1.735559935265399 0loss/SRqqt2/detection_loss‚2.082955307194165 epoch‚6 % _timestamp‚1.7884509239052145e+09 1loss/SRqqt2/assignment_loss‚1.532826676787365 2loss/SRqqt3/assignment_loss‚1.0203677078835287 /loss/FRt4/detection_loss‚2.1706248156049033 0loss/SRqqt3/detection_loss‚1.288214198361009 2loss/SRqqt4/assignment_loss‚1.3685227925756147 .loss/FBt4/detection_loss‚2.489545399492437 /loss/FRt2/assignment_loss‚3.451342338323593 1loss/FBt2/assignment_loss‚0.46206453135335535 /loss/FRt3/detection_loss‚1.8588239978602592 1loss/SRqqt4/detection_loss‚1.4272765500204905ñ 7…¼“ï‚(:ˆ ñ°æÔ¨Øý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.978' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5956 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0_(ùˆnð‚(:ã ñ°æÔ–áÿ% disk./.usageGB‚302.30250549316406 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.drbd2266.in‚0$ memory_percent‚94.20571327209473 disk.dm-1.in‚ 0.01171875 disk.rbd0.out‚0 proc.memory.rssMB‚2642.875* disk./.usagePercent‚20.287774792343072 disk.sda1.in‚0 disk.drbd1716.out‚0 proc.cpu.threads‚17 network.recv‚ 2.166971e+06 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 33.0234375 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚24847.07177734375 disk.dm-0.in‚0.078125* proc.memory.percent‚16.130828857421875 network.sent‚ 1.8229956e+07 disk.sda2.out‚0& disk.nvme0n1.out‚24847.07177734375 disk.dm-1.out‚ 233.0390625 cpu‚25.348666741056366# disk.dm-0.out‚24847.07177734375 disk.sda2.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 233.0390625 disk.drbd2266.out‚0 disk.sda.out‚ 233.0390625 disk.sda1.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0' proc.memory.availableMB‚ 949.3359375û¶„Ò“ñ‚(:ˆ €±æÔ€±˜ý gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚192.428' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.4856 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0åí2-hò‚(:Ý €±æÔØç ÿ network.sent‚ 1.8339145e+07 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚24892.79833984375 disk.sda3.in‚ 0.01171875 disk.drbd1716.in‚0' proc.memory.availableMB‚ 949.3359375 disk.dm-1.out‚ 233.578125 disk.drbd2080.in‚0 disk.drbd2265.in‚0 proc.memory.rssMB‚2642.875* proc.memory.percent‚16.130828857421875 disk.sda2.out‚0 disk.drbd2265.out‚0$ disk./.usageGB‚302.3163070678711 disk.rbd0.out‚0 disk.sda.out‚ 233.578125 disk.dm-1.in‚ 0.01171875 network.recv‚ 2.17401e+06* disk./.usagePercent‚20.288701027601956 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.rbd1.out‚ 33.171875& disk.nvme0n1.out‚24892.79833984375# disk.dm-0.out‚24892.79833984375 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda3.out‚ 233.578125 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0$ memory_percent‚94.20571327209473 cpu‚25.347929710644358 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 proc.cpu.threads‚17áðp“ó‚(:ˆ ±æÔ¥‘ðü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.868' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9736 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0É~Ïoô‚(:ä ±æÔÍ–ôþ disk.dm-1.out‚ 234.12890625 disk.sda1.out‚0 disk.rbd1.in‚0* proc.memory.percent‚16.130828857421875 disk.drbd2265.out‚0 disk.sda.out‚ 234.12890625 cpu‚25.347200643967124 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 proc.memory.rssMB‚2642.875 proc.cpu.threads‚17% disk./.usageGB‚302.33931732177734 disk.sda2.in‚0' disk.nvme0c0n1.out‚24945.2392578125 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 network.sent‚ 1.8344281e+07 network.recv‚ 2.177339e+06 disk.rbd1.out‚ 33.3828125 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0( proc.memory.availableMB‚ 949.32421875* disk./.usagePercent‚20.290245265048494 disk.drbd1716.out‚0 disk.drbd2266.out‚0 disk.sda2.out‚0% disk.nvme0n1.out‚24945.2392578125 disk.drbd2265.in‚0 disk.sda3.out‚ 234.12890625 disk.drbd2266.in‚0" disk.dm-0.out‚24945.2392578125 disk.sda1.in‚0 disk.drbd2080.out‚0$ memory_percent‚94.20578479766846 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0{1Öƒ2õ- _wandbruntime‚6075  _runtime‚6075;”¶ÇöÁ _runtime‚6075.147735859 _step‚626 & lr-AdamW/pg1‚4.531612331113471e-05 & lr-AdamW/pg2‚4.531612331113471e-05 trainer/global_step‚7649 % _timestamp‚1.7884509596762066e+09,Y¶Ì÷Æ _step‚626 & lr-AdamW/pg1‚4.531612331113471e-05 & lr-AdamW/pg2‚4.531612331113471e-05 trainer/global_step‚7649 % _timestamp‚1.7884509596762066e+09 _runtime‚6075.147735859òùÅšý2ø- _wandbruntime‚6075  _runtime‚6075D¨_£æ'ùàO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6409353558016 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 111314666 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 144452148 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚64949412 _runtime‚6075.148654282 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 111314666 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 $ _timestamp‚1.788450959676867e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 255766812 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 111314664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚36621583 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6409353558016 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚445039410559488 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 144472186 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚451448764117504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 255786852 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚36621139 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚451448089752064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚445039392556544 _step‚627 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚433373323258880 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 111314664 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚439758425097144 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6723268622336 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 144472186 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚451448089752064 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚445039410559488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177078272 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚439759096574260 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚4305)ë'úåO M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 144452148 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚64949412 _runtime‚6075.148654282 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 111314666 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 $ _timestamp‚1.788450959676867e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 255766812 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 111314664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚36621583 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6409353558016 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚445039410559488 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 144472186 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚451448764117504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 255786852 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚36621139 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚451448089752064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚445039392556544 _step‚627 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚433373323258880 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 111314664 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚439758425097144 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6723268622336 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 144472186 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚451448089752064 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚445039410559488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 177078272 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚439759096574260 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6409353558016 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 111314666 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360ó²·2û- _wandbruntime‚6075  _runtime‚6075 ŒMi  üêM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚36626087 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 255799282 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 111328682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚451505936597504 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚427 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚439814145662668 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6386572711192 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚486027476443136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 $ _timestamp‚1.788450960365084e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 144490666 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 144470602 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚628 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 144470602 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚486027452503040 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PBDeviceStatsMonitor.on_train_batch_‡èáPend/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6409516623360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚36625660 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6410172999168 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚451505936597504 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚433428244428800 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚433428227389440 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚64957580 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6724296694272 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 144490666 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6409516623360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 111328680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 255799282 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 101583240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚439814817139992 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚445095745595392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚492751773137408 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6724141472256 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 111328682 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚7649 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚6075.837393501 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 101583669 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚492751593975296 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚64957582 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 255819348 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6410172999168 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚445095763598336 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚429 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 255819348 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0"…¢íõ&ýïM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚445095745595392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚492751773137408 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6724141472256 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 111328682 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚7649 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚6075.837393501 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 101583669 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚492751593975296 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚64957582 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 255819348 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6410172999168 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚445095763598336 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚429 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 255819348 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚36626087 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 255799282 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 111328682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚451505936597504 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚427 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚439814145662668 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6386572711192 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚486027476443136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 $ _timestamp‚1.788450960365084e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 144490666 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 144470602 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _step‚628 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 144470602 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚486027452503040 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6409516623360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚36625660 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6410172999168 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚451505936597504 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚433428244428800 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚433428227389440 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚64957580 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6724296694272 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 144490666 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6409516623360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 111328680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 255799282 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 101583240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚439814817139992 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928ô+¨h2þ- _wandbruntime‚6075  _runtime‚6075 ­…UÿÏ /loss/FRt2/assignment_loss‚2.918020008241429 0loss/FRt4/assignment_loss‚3.2447345826936806 0loss/FBt4/assignment_loss‚0.4012919060227432 1loss/SRqqt1/detection_loss‚2.3963050772162044 1loss/SRqqt2/detection_loss‚1.8313972214641778 .loss/FBt1/detection_loss‚4.547046047449112 _step‚629 .loss/FRt1/detection_loss‚2.705705552566342 0loss/FRt3/assignment_loss‚1.8319963202590035 1loss/SRqqt1/assignment_loss‚1.948687064297059 2loss/SRqqt4/assignment_loss‚1.4359172448515891 0loss/FBt3/assignment_loss‚0.2918772233490747 .loss/FRt4/detection_loss‚2.406989195748516 /loss/FBt3/detection_loss‚0.8907083354349579 % _timestamp‚1.7884509603669724e+09 /loss/FRt1/assignment_loss‚4.099274152662696 /loss/FRt3/detection_loss‚2.1191778924493563 /loss/FBt2/detection_loss‚2.4848925828933717 _runtime‚6075.838450996 1loss/SRqqt2/assignment_loss‚1.283764747639551 2loss/SRqqt3/assignment_loss‚0.9628752101870144 .loss/FBt4/detection_loss‚3.214520072937012 &loss/total_loss‚46.964592583238414 0loss/FBt2/assignment_loss‚0.2194722183669261 1loss/SRqqt4/detection_loss‚1.9681296333670617 trainer/global_step‚7649 ,loss/FRt2/detection_loss‚2.3930103393162 0loss/SRqqt3/detection_loss‚1.161709862158579 1loss/FBt1/assignment_loss‚0.20709009166675968 epoch‚6º6¡÷Z€Ô 0loss/FBt2/assignment_loss‚0.2194722183669261 1loss/SRqqt4/detection_loss‚1.9681296333670617 trainer/global_step‚7649 ,loss/FRt2/detection_loss‚2.3930103393162 0loss/SRqqt3/detection_loss‚1.161709862158579 1loss/FBt1/assignment_loss‚0.20709009166675968 epoch‚6 /loss/FRt2/assignment_loss‚2.918020008241429 0loss/FRt4/assignment_loss‚3.2447345826936806 0loss/FBt4/assignment_loss‚0.4012919060227432 1loss/SRqqt1/detection_loss‚2.3963050772162044 1loss/SRqqt2/detection_loss‚1.8313972214641778 .loss/FBt1/detection_loss‚4.547046047449112 _step‚629 .loss/FRt1/detection_loss‚2.705705552566342 0loss/FRt3/assignment_loss‚1.8319963202590035 1loss/SRqqt1/assignment_loss‚1.948687064297059 2loss/SRqqt4/assignment_loss‚1.4359172448515891 0loss/FBt3/assignment_loss‚0.2918772233490747 .loss/FRt4/detection_loss‚2.406989195748516 /loss/FBt3/detection_loss‚0.8907083354349579 % _timestamp‚1.7884509603669724e+09 /loss/FRt1/assignment_loss‚4.099274152662696 /loss/FRt3/detection_loss‚2.1191778924493563 /loss/FBt2/detection_loss‚2.4848925828933717 _runtime‚6075.838450996 1loss/SRqqt2/assignment_loss‚1.283764747639551 2loss/SRqqt3/assignment_loss‚0.9628752101870144 .loss/FBt4/detection_loss‚3.214520072937012 &loss/total_loss‚46.964592583238414õY™Ç†Á % _timestamp‚1.7884509966483228e+09 _runtime‚6112.120015681 _step‚630 & lr-AdamW/pg1‚4.444141803222954e-05 & lr-AdamW/pg2‚4.444141803222954e-05 trainer/global_step‚7699™º&yÌ‡Æ & lr-AdamW/pg1‚4.444141803222954e-05 & lr-AdamW/pg2‚4.444141803222954e-05 trainer/global_step‚7699 % _timestamp‚1.7884509966483228e+09 _runtime‚6112.120015681 _step‚630öS¯[t2ˆ- _wandbruntime‚6112  _runtime‚61120±°q%‰ßO 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 102208754 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚454306727500288 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _step‚631 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚495814879767040 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚489048338648576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _runtime‚6112.120811102 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚447857062501888 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6449683001344 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 112015464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚454306727500288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚454 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6766517178368 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚36850940 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 trainer/global_step‚7699 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 257413704 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚36850488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚442544466964552 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 257393664 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚447857044498944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 112015466 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6449683001344 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚452 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚436118546826240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚454307401865728 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚442545138441668 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 145398238 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚65357814 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 145398238 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚495814698494464 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6450339363840 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚65357812 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6449683001344 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚452 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚436118546826240 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚454307401865728 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚442545138441668 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 145398238 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚65357814 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 145398238 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚495814698494464 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6450339363840 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚65357812 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚454306727500288 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _step‚631 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚495814879767040 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚489048338648576 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _runtime‚6112.120811102 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚447857062501888 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6449683001344 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 112015464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0÷'ØÂ>2‹- _wandbruntime‚6112  _runtime‚6112eeÐ ñ&ŒëM B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚36855513 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 102221034 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚447913415540736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 257426134 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚442600187530076 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚489110013326336 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6767210152448 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 145416718 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6451158804992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚489109989386240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚436173450956800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6451158804992 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 145396654 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚495877199538688 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 102221495 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 257446200 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚442600859007400 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6450502429184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 % _timestamp‚1.7884509973748772e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6427408050600 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚65365980 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚447913415540736 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 112029480 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 257426134 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 257446200 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚00 ñ¼ö&ðM FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚65365980 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚447913415540736 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 112029480 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 257426134 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 257446200 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚36855513 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 102221034 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚447913415540736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 257426134 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚442600187530076 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚489110013326336 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6767210152448 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 145416718 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6451158804992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚489109989386240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚436173450956800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6451158804992 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 145396654 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚495877199538688 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 102221495 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 257446200 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚442600859007400 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6450502429184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 % _timestamp‚1.7884509973748772e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6427408050600 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872ø»Aiá2Ž- _wandbruntime‚6112  _runtime‚6112ç% ^Ø epoch‚6 2loss/SRqqt1/assignment_loss‚1.9500890334861145 1loss/SRqqt3/detection_loss‚1.0839843767792432 %loss/total_loss‚42.87834290834535 trainer/global_step‚7699 _step‚633 0loss/FRt1/assignment_loss‚3.9743386092393296 /loss/FRt1/detection_loss‚3.2280607715896936 /loss/FBt1/detection_loss‚2.6558456420898438 .loss/FRt2/detection_loss‚2.118458268104815 1loss/SRqqt4/detection_loss‚1.5238671572258076 0loss/FRt4/assignment_loss‚2.4388307788155297 2loss/SRqqt4/assignment_loss‚1.3009967194559673 0loss/FBt4/assignment_loss‚0.3868802870998219 /loss/FRt4/detection_loss‚2.1265607622536744 0loss/FRt2/assignment_loss‚3.4584636442801533 1loss/FBt1/assignment_loss‚0.47864434350620183 .loss/FRt3/detection_loss‚1.689930173026961 1loss/SRqqt2/detection_loss‚1.7327393072728927 _runtime‚6112.848125391 2loss/SRqqt2/assignment_loss‚1.8557958067036593 /loss/FBt3/detection_loss‚0.6524244645701313 0loss/FRt3/assignment_loss‚1.5139958775320719 2loss/SRqqt3/assignment_loss‚1.2377211778744388 1loss/FBt3/assignment_loss‚0.16854093472711107 1loss/SRqqt1/detection_loss‚2.3144120375315347 /loss/FBt2/detection_loss‚1.9014052137732507 .loss/FBt4/detection_loss‚2.435794336455209 % _timestamp‚1.7884509973766158e+09 0loss/FBt2/assignment_loss‚0.65056318495189770‹ÁÝ .loss/FBt4/detection_loss‚2.435794336455209 % _timestamp‚1.7884509973766158e+09 0loss/FBt2/assignment_loss‚0.6505631849518977 epoch‚6 2loss/SRqqt1/assignment_loss‚1.9500890334861145 1loss/SRqqt3/detection_loss‚1.0839843767792432 %loss/total_loss‚42.87834290834535 trainer/global_step‚7699 _step‚633 0loss/FRt1/assignment_loss‚3.9743386092393296 /loss/FRt1/detection_loss‚3.2280607715896936 /los… ßO¢s/FBt1/detection_loss‚2.6558456420898438 .loss/FRt2/detection_loss‚2.118458268104815 1loss/SRqqt4/detection_loss‚1.5238671572258076 0loss/FRt4/assignment_loss‚2.4388307788155297 2loss/SRqqt4/assignment_loss‚1.3009967194559673 0loss/FBt4/assignment_loss‚0.3868802870998219 /loss/FRt4/detection_loss‚2.1265607622536744 0loss/FRt2/assignment_loss‚3.4584636442801533 1loss/FBt1/assignment_loss‚0.47864434350620183 .loss/FRt3/detection_loss‚1.689930173026961 1loss/SRqqt2/detection_loss‚1.7327393072728927 _runtime‚6112.848125391 2loss/SRqqt2/assignment_loss‚1.8557958067036593 /loss/FBt3/detection_loss‚0.6524244645701313 0loss/FRt3/assignment_loss‚1.5139958775320719 2loss/SRqqt3/assignment_loss‚1.2377211778744388 1loss/FBt3/assignment_loss‚0.16854093472711107 1loss/SRqqt1/detection_loss‚2.3144120375315347 /loss/FBt2/detection_loss‚1.9014052137732507ùïxEù“‘‚(:ˆ ¼±æÔŠ”äü gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.078' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.4156 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Î"„’Z’‚(:Ï ¼±æÔª¾ÿþ# disk./.usageGB‚302.392520904541 disk.rbd1.in‚0 disk.nvme0c0n1.out‚ 25097.625 disk.rbd0.in‚0 disk.sda.out‚ 235.81640625' proc.memory.availableMB‚ 1024.078125 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 network.sent‚ 1.84652e+07 disk.drbd1716.in‚0 disk.sda.in‚ 0.01171875* disk./.usagePercent‚20.293815802128538 disk.nvme0n1.out‚ 25097.625 disk.dm-1.out‚ 235.81640625 disk.sda3.in‚ 0.01171875 disk.dm-0.out‚ 25097.625 network.recv‚ 2.19285e+06 disk.drbd2265.out‚0 disk.drbd2080.out‚0# memory_percent‚93.7495231628418# proc.memory.rssMB‚ 2568.16796875 disk.sda3.out‚ 235.81640625 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0 proc.cpu.threads‚17 disk.rbd1.out‚ 33.4775390625 disk.drbd2080.in‚0* proc.memory.percent‚15.674853324890137 cpu‚25.34502505331656Òø›”“‚(:’ ˱æÔ–É•ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.62' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.724000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0{l–s”‚(:è ˱æÔ¬úŽÿ disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 cpu‚25.34430600595269 disk.sda1.out‚0 disk.drbd2266.out‚0 network.sent‚ 1.8573005e+07* disk./.usagePercent‚20.293258217718964( disk.nvme0c0n1.out‚25150.64599609375 disk.drbd2265.in‚0# proc.memory.rssMB‚ 2568.16796875 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 236.265625 disk.drbd2080.in‚0 disk.drbd2080.out‚0* proc.memory.percent‚15.674853324890137 disk.sda.in‚ 0.01171875 disk.sda1.in‚0$ memory_percent‚93.75102519989014 disk.drbd2265.out‚0 disk.rbd1.in‚0& disk.nvme0n1.out‚25150.64599609375 disk.dm-1.out‚ 236.265625 disk.rbd0.in‚0$ disk./.usageGB‚302.3842124938965 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875# disk.dm-0.out‚25150.64599609375 disk.rbd1.out‚ 33.6025390625 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 network.recv‚ 2.198315e+06 disk.sda3.out‚ 236.265625 disk.sda2.out‚0 disk.drbd1716.out‚0) proc.memory.availableMB‚ 1023.83203125 disk.rbd0.out‚0ì½<Õ2•- _wandbruntime‚6148  _runtime‚6148DñÈ–Â ' lr-AdamW/pg2‚4.3570651782661376e-05 trainer/global_step‚7749 % _timestamp‚1.7884510334635208e+09 _runtime‚ 6148.93509749 _step‚634 ' lr-AdamW/pg1‚4.3570651782661376e-05ŽJÃèÍ—Ç ' lr-AdamW/pg1‚4.3570651782661376e-05 ' lr-AdamW/pg2‚4.3570651782661376e-05 trainer/global_step‚7749 % _timestamp‚1.7884510334635208e+09 _runtime‚ 6148.93509749 _step‚634únIp«2˜- _wandbruntime‚6148  _runtime‚6148+Íåç'™áO ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚450674714444288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 112716264 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚498940153291264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6467426954836 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _step‚635 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚457166039613952 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 259020516 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 112716264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚450674714444288 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 102845483 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 146304252 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚6148.936204489 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6491325169664 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 146324290 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚492130875531776 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚450674696441344 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚438863753354240 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 112716266 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 112716266 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 102845027 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚65766214 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 146324290 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 259040556 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚457165365248512 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚457165365248512 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 146304252 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚450674696441344 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 trainer/global_step‚7749 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚65766212 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚457165365248512 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 146304252 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚450674696441344 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 trainer/global_step‚7749 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚65766212 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚438863753354240 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 112716266 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 112716266 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 102845027 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0ûÊ5‰ó2›- _wandbruntime‚6149  _runtime‚6149šäÝœëM E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚438918657484800 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 259052986 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚445386900874808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6491488235008 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6810295921152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚464 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚457222537715200 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚462 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 102857753 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 112730282 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 259073052 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚457223212093952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 259073052 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 146322706 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 146322706 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6492144610816 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚450731049480192 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚37083835 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6810119727616 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 259052986 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚499002846130688 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 trainer/global_step‚7749 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6468243390008 _runtime‚6149.640732487 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚450731067483136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 146342770 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 % _timestamp‚1.7884510341687138e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 112730282 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚499002645997056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚457223212093952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚65774380 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚37083373 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚636 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6467588952044 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚450731049480192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚65774382 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚492192550209536,¶Oö&ðM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚65774382 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚492192550209536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚438918657484800 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 259052986 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚445386900874808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6491488235008 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6810295921152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚464 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚457222537715200 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚462 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 102857753 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 112730282 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 259073052 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚457223212093952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 259073052 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 146322706 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 146322706 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6492144610816 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚450731049480192 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚37083835 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6810119727616 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 259052986 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚499002846130688 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 trainer/global_step‚7749 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6468243390008 _runtime‚6149.640732487 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚450731067483136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 146342770 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 % _timestamp‚1.7884510341687138e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 112730282 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚499002645997056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚457223212093952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚65774380 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚37083373 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚636 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6467588952044 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚450731049480192üV¬",2ž- _wandbruntime‚6149  _runtime‚6149s8×nXŸÒ %loss/total_loss‚44.33662609824208 .loss/FBt1/detection_loss‚2.477636218070984 _runtime‚6149.642018804 .loss/FBt4/detection_loss‚3.316255807876587 /loss/FRt2/detection_loss‚1.9769613283111693 1loss/SRqqt4/assignment_loss‚1.299585812655625 /loss/FRt1/detection_loss‚2.9136555455625057 0loss/SRqqt1/detection_loss‚2.485912318740572 1loss/SRqqt2/assignment_loss‚1.302598724068672 2loss/SRqqt3/assignment_loss‚0.9407916343625402 /loss/FRt3/detection_loss‚1.7582362161441283 trainer/global_step‚7749 /loss/FRt4/detection_loss‚2.1638927036630258 epoch‚6 % _timestamp‚1.7884510341705015e+09 _step‚637 1loss/FBt4/assignment_loss‚0.20167961716651917 1loss/SRqqt4/detection_loss‚1.4498618820042752 /loss/FRt1/assignment_loss‚4.457894708961248 /loss/FRt4/assignment_loss‚2.720103863094534 2loss/SRqqt1/assignment_loss‚2.0697935351303647 1loss/SRqqt3/detection_loss‚1.1018835781142116 0loss/FRt2/assignment_loss‚3.6874121414854173 .loss/FBt1/assignment_loss‚0.63110052581281 /loss/FRt3/assignment_loss‚1.677891819043593 1loss/FBt2/assignment_loss‚0.49414776159180207 1loss/FBt3/assignment_loss‚0.18955889980522458 1loss/SRqqt2/detection_loss‚1.9643980290959864 /loss/FBt2/detection_loss‚2.2359452013458525 /loss/FBt3/detection_loss‚0.8194282261344293û`NO] × _step‚637 1loss/FBt4/assignment_loss‚0.20167961716651917 1loss/SRqqt4/detection_loss‚1.4498618820042752 /loss/FRt1/assignment_loss‚4.457894708961248 /loss/FRt4/assignment_loss‚2.720103863094534 2loss/SRqqt1/assignment_loss‚2.0697935351303647 1loss/SRqqt3/detection_loss‚1.1018835781142116 0loss/FRt2/assignment_loss‚3.6874121414854173 .loss/FBt1/assignment_loss‚0.63110052581281 /loss/FRt3/assignment_loss‚1.677891819043593 1loss/FBt2/assignment_loss‚0.49414776159180207 1loss/FBt3/assignment_loss‚0.18955889980522458 1loss/SRqqt2/detection_loss‚1.9643980290959864 /loss/FBt2/detection_loss‚2.2359452013458525 /loss/FBt3/detection_loss‚0.8194282261344293 %loss/total_loss‚44.33662609824208 .loss/FBt1/detection_loss‚2.477636218070984 _runtime‚6149.642018804 .loss/FBt4/detection_loss‚3.316255807876587 /loss/FRt2/detection_loss‚1.9769613283111693 1loss/SRqqt4/assignment_loss‚1.299585812655625 /loss/FRt1/detection_loss‚2.9136555455625057 0loss/SRqqt1/detection_loss‚2.485912318740572 1loss/SRqqt2/assignment_loss‚1.302598724068672 2loss/SRqqt3/assignment_loss‚0.9407916343625402 /loss/FRt3/detection_loss‚1.7582362161441283 trainer/global_step‚7749 /loss/FRt4/detection_loss‚2.1638927036630258 epoch‚6 % _timestamp‚1.7884510341705015e+09ýˆ·”ð“¡‚(:ˆ Ú±æÔÎõ¨ý gpu.0.gpu‚41.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚182.362' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4724 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ùlJ{¢‚(:ð Ú±æÔ¬ú¯€ proc.cpu.threads‚17 disk.dm-1.in‚ 0.01171875 network.sent‚ 1.8579511e+07 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚25178.27099609375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 network.recv‚ 2.202565e+06 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 237.08984375 disk.sda2.in‚0% disk./.usageGB‚302.39663314819336 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 33.6650390625# proc.memory.rssMB‚ 2568.16796875 disk.sda3.out‚ 237.08984375) proc.memory.availableMB‚ 1023.79296875& disk.nvme0n1.out‚25178.27099609375 disk.dm-1.out‚ 237.08984375 disk.rbd0.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.rbd1.in‚0# disk.dm-0.out‚25178.27099609375 disk.sda.in‚ 0.01171875* proc.memory.percent‚15.674853324890137* disk./.usagePercent‚20.294091778250454 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0$ memory_percent‚93.75126361846924 cpu‚25.343540446119416#6µn“£‚(:ˆ é±æÔêàžý gpu.0.gpu‚42.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚175.158' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.0316 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ºY,¶e¤‚(:Ú é±æÔûŬÿ network.sent‚ 1.8686844e+07% disk./.usageGB‚302.34081649780273 disk.sda1.out‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 33.6650390625 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125) proc.memory.availableMB‚ 1024.29296875 disk.sda.out‚ 237.46875 disk.drbd1716.in‚0# proc.memory.rssMB‚ 2568.16796875* proc.memory.percent‚15.674853324890137 disk.dm-0.in‚0.078125 disk.dm-1.in‚ 0.01171875 network.recv‚ 2.208338e+06 disk.sda1.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 disk.dm-0.out‚ 25243.7109375 disk.sda.in‚ 0.01171875 disk.dm-1.out‚ 237.46875 disk.drbd2265.out‚0 disk.sda3.out‚ 237.46875 disk.rbd0.in‚0$ memory_percent‚93.74821186065674 proc.cpu.threads‚17 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125" disk.nvme0n1.out‚ 25243.7109375 disk.drbd2265.in‚0 disk.drbd2266.out‚0 cpu‚25.34287865415385* disk./.usagePercent‚20.290345876009447$ disk.nvme0c0n1.out‚ 25243.7109375 disk.drbd2266.in‚0© k‘¥‚(:’ ø±æÔ¶¡Šý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.95' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.989999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0A m¦‚(:â ø±æÔСŸÿ* proc.memory.percent‚15.674853324890137 disk.drbd2080.out‚0 disk.dm-0.in‚0.078125 disk.drbd2265.in‚0 disk.rbd0.out‚0* disk./.usagePercent‚20.292483538920354 disk.sda.in‚ 0.01171875 disk.sda1.in‚0# proc.memory.rssMB‚ 2568.16796875 network.recv‚ 2.213264e+06 disk.sda3.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875) proc.memory.availableMB‚ 1024.05078125 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125$ disk./.usageGB‚302.3726692199707 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 network.sent‚ 1.8693424e+07 disk.sda.out‚ 238.07421875 disk.sda1.out‚0$ memory_percent‚93.74969005584717 cpu‚25.342131793085603 disk.sda3.out‚ 238.07421875 disk.rbd1.out‚ 33.697265625" disk.nvme0n1.out‚ 25292.2578125 disk.drbd2265.out‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17 disk.dm-1.out‚ 238.07421875 disk.nvme0c0n1.in‚0.078125$ disk.nvme0c0n1.out‚ 25292.2578125 disk.dm-0.out‚ 25292.2578125 disk.drbd1716.in‚0 disk.drbd1716.out‚0죲2§- _wandbruntime‚6186  _runtime‚6186€çê}ƨÀ _step‚638 & lr-AdamW/pg1‚4.270400602881153e-05 & lr-AdamW/pg2‚4.270400602881153e-05 trainer/global_step‚7799 $ _timestamp‚1.788451070674006e+09 _runtime‚6186.145577445O¦¦OË©Å & lr-AdamW/pg1‚4.270400602881153e-05 & lr-AdamW/pg2‚4.270400602881153e-05 trainer/global_step‚7799 $ _timestamp‚1.788451070674006e+09 _runtime‚6186.145577445 _step‚638þnWþy2ª- _wandbruntime‚6186  _runtime‚6186ðZHç'«áO LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 113417064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚448116550699368 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚441608959882240 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚460024677362176 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6532310975488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 147250342 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 147230304 _runtime‚6186.146787238 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6531654612992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚37306769 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚495213412414976 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 trainer/global_step‚7799 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 103481827 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚66174614 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 113417064 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚502065765155328 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 147230304 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚495213436355072 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚66174612 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚453492366386688 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6532310975488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚460024002996736 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚460024002996736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6508262294244 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚453492366386688 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚453492348383744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6852518461440 _step‚639 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7884510706748767e+09 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 113417066 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚444 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚460024677362176 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚502065954816512 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6852352740352 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚441608942842880 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 113417066 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 260667408 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 113417064 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚502065765155328 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 147230ÈÚÄ;304 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚495213436355072 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚66174612 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚453492366386688 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6532310975488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚460024002996736 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚460024002996736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6508262294244 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚453492366386688 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚453492348383744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6852518461440 _step‚639 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7884510706748767e+09 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 113417066 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚444 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚460024677362176 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚502065954816512 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6852352740352 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚441608942842880 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 113417066 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 260667408 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6532310975488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 147250342 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 147230304ÿ•–”c2­- _wandbruntime‚6186  _runtime‚61862v/ð&®êM C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 260699904 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚453548719425536 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 147268822 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 260679838 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 260679838 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 113431082 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 260699904 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 113431082 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 $ _timestamp‚1.788451071364848e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 147268822 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 103494160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚453548701422592 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _step‚640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚460081175463424 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚460081849842176 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 113431080 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6508424291452 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6532474040832 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 103494601 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 147248758 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚66182780 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚441663846973440 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚460081849842176 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 113431080 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚37311380 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6853380719104 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 260699904 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚453548719425536 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 147268822 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 260679838 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 260679838 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 113431082 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 260699904 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 113431082 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 $ _timestamp‚1.788451071364848e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 147268822 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 103494160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚453548701422592 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _step‚640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚460081175463424 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚460081849842176 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 113431080 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6508424291452 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6532474040832 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 103494601 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 147248758 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚66182780 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚441663846973440 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚460081849842176 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 113431080 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚37311380 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6853380719104 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚450903264043892 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 114117864 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 148176394 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚37536334 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6572640418816 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚505191443549696 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6549097633652 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _step‚643 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 262274220 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6895470311424 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚462883315110400 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6572640418816 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 114117866 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 148156356 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6573296781312 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚456310000326144 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚462882640744960 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 104119805 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚444354166410240 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6548443195896 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚498295949298176 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 % _timestamp‚1.7884511072346187e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚462883315110400 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 114117864 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚505191260179968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424é3ñì'¼æO UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚462882640744960 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 104119805 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚444354166410240 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6548443195896 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚498295949298176 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 % _timestamp‚1.7884511072346187e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚462883315110400 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 114117864 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚505191260179968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 148156356 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚456310018329088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6573296781312 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 262294260 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 114117866 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚450903264043892 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 114117864 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 148176394 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚37536334 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6572640418816 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚505191443549696 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6549097633652 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _step‚643 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 262274220 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6895470311424 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚462883315110400 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6572640418816 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 114117866 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 148156356 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6573296781312 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚456310000326144ƒà, 2½- _wandbruntime‚6223  _runtime‚6223íADôñ&¾ëM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 176193536 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚456366353364992 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚505253952264704 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 262326756 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 148174810 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚66591182 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚450958313132300 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚456366353364992 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6573459846656 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚37541315 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6896152095232 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6574116222464 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 148174810 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 % _timestamp‚1.7884511079162657e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 262306690 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚37540857 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚462939813211648 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚456366371367936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 262306690 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 114131882 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 104132037 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚498357623975936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚456366371367936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6574116222464 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200133632 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚444409053501440 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 262326756 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6549914068824 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚462940487590400 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚505253752131072 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚66591180 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6549259630860 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6896328288768 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚460 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 114131880 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚462940487590400 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚450958984609624 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚444409070540800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚6223.388363506 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 114131882 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844_‹Dh¿ðM REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚66591180 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6549259630860 P@DeviceStatsMonitor.on_train_batch_eü9¦Ž$nd/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6896328288768 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚460 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 114131880 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚462940487590400 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚450958984609624 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚444409070540800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚6223.388363506 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 114131882 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 176193536 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚456366353364992 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚505253952264704 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 262326756 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 148174810 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚66591182 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚450958313132300 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚456366353364992 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6573459846656 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚37541315 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6896152095232 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6574116222464 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 148174810 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 % _timestamp‚1.7884511079162657e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 262306690 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚37540857 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚462939813211648 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚456366371367936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 262306690 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 114131882 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 104132037 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚498357623975936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚456366371367936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6574116222464 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200133632 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚444409053501440 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 262326756 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6549914068824 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚462940487590400 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚505253752131072 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844„“p½£2À- _wandbruntime‚6223  _runtime‚6223VŸ(i\ÁÖ 1loss/SRqqt1/assignment_loss‚1.731766153126955 1loss/SRqqt2/detection_loss‚1.9832480642944574 1loss/SRqqt4/detection_loss‚1.4955066286267773 .loss/FBt1/detection_loss‚4.032285292943318 0loss/FBt4/assignment_loss‚0.6116475413242976 .loss/FRt1/detection_loss‚3.158513811727365 %loss/total_loss‚45.58375636126513 epoch‚6 % _timestamp‚1.7884511079183428e+09 0loss/FRt4/assignment_loss‚3.2697418332099915 2loss/SRqqt2/assignment_loss‚1.4192566685378551 1loss/FBt1/assignment_loss‚0.09887931418230285 /loss/FRt3/detection_loss‚1.6645675489535698 /loss/FRt4/detection_loss‚2.4148791118763215 1loss/SRqqt3/detection_loss‚0.9946566937785399 /loss/FBt2/detection_loss‚1.9032520109415054 1loss/FBt3/assignment_loss‚0.19679946894856085 .loss/FRt2/detection_loss‚2.153264819536098 1loss/SRqqt1/detection_loss‚3.0638254553079607 /loss/FBt3/detection_loss‚0.6289178565324794 _runtime‚6223.389861893 0loss/FRt2/assignment_loss‚2.8635930876399196 _step‚645 0loss/FRt1/assignment_loss‚4.5541495978832245 0loss/FRt3/assignment_loss‚1.0145062419084403 0loss/FBt2/assignment_loss‚0.8117352380603546 2loss/SRqqt3/assignment_loss‚1.0552991370082294 2loss/SRqqt4/assignment_loss‚1.5398333552443693 .loss/FBt4/detection_loss‚2.923631429672241 trainer/global_step‚7849' ûaÂÛ 1loss/SRqqt3/detection_loss‚0.9946566937785399 /loss/FBt2/detection_loss‚1.9032520109415054 1loss/FBt3/assignment_loss‚0.19679946894856085 .loss/FRt2/detection_loss‚2.153264819536098 1loss/SRqqt1/detection_loss‚3.0638254553079607 /loss/FBt3/detection_loss‚0.6289178565324794 _runtime‚6223.389861893 0loss/FRt2/assignment_loss‚2.8635930876399196 _step‚645 0loss/FRt1/assignment_loss‚4.5541495978832245 0loss/FRt3/assignment_loss‚1.0145062419084403 0loss/FBt2/assignment_loss‚0.8117352380603546 2loss/SRqqt3/assignment_loss‚1.0552991370082294 2loss/SRqqt4/assignment_loss‚1.5398333552443693 .loss/FBt4/detection_loss‚2.923631429672241 trainer/global_step‚7849 1loss/SRqqt1/assignment_loss‚1.731766153126955 1loss/SRqqt2/detection_loss‚1.9832480642944574 1loss/SRqqt4/detection_loss‚1.4955066286267773 .loss/FBt1/detection_loss‚4.032285292943318 0loss/FBt4/assignment_loss‚0.6116475413242976 .loss/FRt1/detection_loss‚3.158513811727365 %loss/total_loss‚45.58375636126513 epoch‚6 % _timestamp‚1.7884511079183428e+09 0loss/FRt4/assignment_loss‚3.2697418332099915 2loss/SRqqt2/assignment_loss‚1.4192566685378551 1loss/FBt1/assignment_loss‚0.09887931418230285 /loss/FRt3/detection_loss‚1.6645675489535698 /loss/FRt4/detection_loss‚2.4148791118763215…vvážÃ‚(:“ ¥²æÔ‚ü¡ý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.374' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.474799999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€âWqÄ‚(:æ ¥²æÔ¹Š¦€ disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚25455.1240234375 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 cpu‚25.3399810445898$ disk./.usageGB‚302.4027786254883 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-1.out‚ 239.58984375$ memory_percent‚92.16475486755371( proc.memory.availableMB‚ 1283.7265625 network.sent‚ 1.881298e+07 disk.rbd0.out‚0" disk.dm-0.out‚25455.1240234375 disk.nvme0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 33.884765625 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 network.recv‚ 2.22775e+06 disk.sda1.in‚0 disk.rbd1.in‚0 disk.sda3.out‚ 239.58984375 disk.sda.out‚ 239.58984375 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875* proc.memory.percent‚15.674853324890137* disk./.usagePercent‚20.294504206388115 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0% disk.nvme0n1.out‚25455.1240234375 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.sda1.out‚0 disk.drbd1716.out‚0# proc.memory.rssMB‚ 2568.16796875=T”P“Å‚(:ˆ ´²æÔÎõ¤ý gpu.0.gpu‚42.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚190.329' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.0658 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¾J‡ sÆ‚(:è ´²æÔŠÀÿ network.sent‚ 1.8922019e+07 disk.drbd2266.in‚0$ disk./.usageGB‚302.4154052734375 disk.rbd1.in‚0 disk.rbd0.in‚0" disk.dm-0.out‚25499.6630859375 disk.sda2.out‚0$ memory_percent‚92.16489791870117 disk.drbd2266.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.sda3.out‚ 240.05859375 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.rbd1.out‚ 33.9169921875% disk.nvme0n1.out‚25499.6630859375 disk.sda1.in‚0* proc.memory.percent‚15.674853324890137) disk./.usagePercent‚20.29535159132645 disk.dm-1.out‚ 240.05859375 disk.drbd1716.in‚0 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚25499.6630859375 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.sda.out‚ 240.05859375 network.recv‚ 2.23414e+06 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125# proc.memory.rssMB‚ 2568.16796875 cpu‚25.339293848200253 proc.cpu.threads‚17 disk.drbd2265.out‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda1.out‚0' proc.memory.availableMB‚ 1283.703125Ë@•‘Ç‚(:† òæÔˆêŸý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚131.77' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.354 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0:Äc/wÈ‚(:ì òæÔÅì¢ÿ disk.dm-1.out‚ 240.51953125 disk.drbd2265.in‚0 disk.sda3.out‚ 240.51953125 disk.sda1.in‚0# proc.memory.rssMB‚ 2568.16796875 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.sda.out‚ 240.51953125 proc.cpu.threads‚17) disk./.usagePercent‚20.29592786428602 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125( disk.nvme0c0n1.out‚25552.02490234375 disk.rbd1.in‚0 network.sent‚ 1.892727e+07 network.recv‚ 2.237478e+06 disk.drbd2265.out‚0 disk.dm-0.in‚0.078125* proc.memory.percent‚15.674853324890137$ disk./.usageGB‚302.4239921569824 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚92.16341972351074 disk.rbd0.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 1283.9453125 cpu‚25.338525660319213& disk.nvme0n1.out‚25552.02490234375 disk.sda.in‚ 0.01171875 disk.sda2.out‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0# disk.dm-0.out‚25552.02490234375 disk.rbd1.out‚ 34.0419921875!kd2É- _wandbruntime‚6259  _runtime‚6259 h@3ÉÊà ' lr-AdamW/pg1‚4.0983797542627796e-05 ' lr-AdamW/pg2‚4.0983797542627796e-05 trainer/global_step‚7899 % _timestamp‚1.7884511436909254e+09 _runtime‚6259.162512108 _step‚646®ïÕoÎËÈ ' lr-AdamW/pg1‚4.0983797542627796e-05 ' lr-AdamW/pg2‚4.0983797542627796e-05 trainer/global_step‚7899 % _timestamp‚1.7884511436909254e+09 _runtime‚6259.162512108 _step‚646†½òÏÃ2Ì- _wandbruntime‚6259  _runtime‚6259l4BÝç'ÍáO `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚459127670271488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6938285968384 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚459127670271488 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 trainer/global_step‚7899 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 114818664 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 149102446 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 263921112 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6938457980928 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6614282587136 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚501378510121472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚647 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6614282587136 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6613626224640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6613626224640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚66991414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚37765298 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚508316968102400 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6589278535304 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚459127652268544 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 114818666 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚447099372938240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 114818666 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 149082408 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚66991412 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 114818664 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚465741278493184 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚447099355898880 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 263901072 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚508316772149760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚459127652268544 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚6259.163432664 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 104756258 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚501378486181376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚465741952858624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 149102446 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚465741278493184 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6589932973060 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚453688634434184ÌÅÏaÎæO N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 104756258 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚501378486181376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚465741952858624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 149102446 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚465741278493184 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6589932973060 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚453688634434184 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚459127670271488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 6938285968384 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚459127670271488 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 trainer/global_step‚7899 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 114818664 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 149102446 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 263921112 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6938457980928 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6614282587136 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚501378510121472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚647 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6614282587136 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6613626224640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6613626224640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚66991414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚37765298 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚508316968102400 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6589278535304 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freedHcؾØ‚459127652268544 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 114818666 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚447099372938240 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 114818666 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 149082408 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚66991412 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 114818664 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚465741278493184 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚447099355898880 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 263901072 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚508316772149760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚459127652268544 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚6259.163432664‡É…V‰2Ï- _wandbruntime‚6259  _runtime‚6259'¶4Äñ&ÐëM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚459184023310336 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6590749408232 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 114832682 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚459184005307392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚508379476391424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚508379297229312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6615102028288 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚465799125338624 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 263953608 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 114832682 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚501440160859136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 263933542 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 149100862 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 104769421 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6590094970268 % _timestamp‚1.7884511443677588e+09 _runtime‚6259.839788623 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6615102028288 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _step‚648 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚66999580 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6614445652480 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚453745026477032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 149120926 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 149120926 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚453744354999708 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚447154260029440 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 trainer/global_step‚7899 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚465799125338624 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6939160310272 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚459184005307392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚501440136919040 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚447154277068800 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚465798450959872 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 263953608 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 114832680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚37769839 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 114832680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 104768971 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚37769391 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚459184023310336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6614445652480 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 263953608 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 114832680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚37769839 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 114832680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 104768971 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚37769391 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚459184023310336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6614445652480 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚459184023310336 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6590749408232 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 114832682 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚459184005307392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚508379476391424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚508379297229312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6615102028288 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚465799125338624 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 263953608 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 114832682 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚501440160859136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 263933542 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 149100862 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 104769421 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6590094970268 % _timestamp‚1.7884511443677588e+09 _runtime‚6259.839788623 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6615102028288 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _step‚648 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚66999580 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6614445652480 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚453745026477032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 149120926 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 149120926 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450ˆ6­¥o2Ò- _wandbruntime‚6259  _runtime‚6259W0Ò¼UÓÏ /loss/FRt3/assignment_loss‚1.492348800345165 0loss/FRt4/assignment_loss‚2.6507799095577664 1loss/SRqqt3/assignment_loss‚1.104753865323229 0loss/FBt2/assignment_loss‚0.4102786245439625 0loss/FBt4/assignment_loss‚0.7940609082579613 /loss/FBt3/detection_loss‚0.7020838116843473 _runtime‚6259.840754775 _step‚649 1loss/SRqqt4/detection_loss‚1.7045143493092978 .loss/FRt1/detection_loss‚2.900332054784221 0loss/SRqqt3/detection_loss‚1.181795587497098 %loss/total_loss‚45.81994117825418 1loss/FBt1/assignment_loss‚0.46663053160481327 1loss/FBt3/assignment_loss‚0.34099090937073434 0loss/SRqqt1/detection_loss‚2.890392998090157 -loss/FBt1/detection_loss‚2.60163941520911 /loss/FBt4/detection_loss‚3.8964579701423645 trainer/global_step‚7899 0loss/FRt2/assignment_loss‚3.1565333916382357 .loss/FRt2/detection_loss‚2.081067446957935 % _timestamp‚1.7884511443692672e+09 /loss/FRt1/assignment_loss‚3.764125966256665 2loss/SRqqt2/assignment_loss‚1.4948940288920243 0loss/SRqqt2/detection_loss‚1.813355294557718 epoch‚6 2loss/SRqqt1/assignment_loss‚1.8282225452936613 2loss/SRqqt4/assignment_loss‚1.3853631551162555 /loss/FRt3/detection_loss‚2.2345839081014076 -loss/FRt4/detection_loss‚2.31556660996543 /loss/FBt2/detection_loss‚2.6091690957546234q—,µZÔÔ 2loss/SRqqt2/assignment_loss‚1.4948940288920243 0loss/SRqqt2/detection_loss‚1.813355294557718 epoch‚6 2loss/SRqqt1/assignment_loss‚1.8282225452936613 2loss/SRqqt4/assignment_loss‚1.3853631551162555 /loss/FRt3/detection_loss‚2.2345839081014076 -loss/FRt4/detection_loss‚2.31556660996543 /loss/FBt2/detection_loss‚2.6091690957546234 /loss/FRt3/assignment_loss‚1.492348800345165 0loss/FRt4/assignment_loss‚2.6507799095577664 1loss/SRqqt3/assignment_loss‚1.104753865323229 0loss/FBt2/assignment_loss‚0.4102786245439625 0loss/FBt4/assignment_loss‚0.7940609082579613 /loss/FBt3/detection_loss‚0.7020838116843473 _runtime‚6259.840754775 _step‚649 1loss/SRqqt4/detection_loss‚1.7045143493092978 .loss/FRt1/detection_loss‚2.900332054784221 0loss/SRqqt3/detection_loss‚1.181795587497098 %loss/total_loss‚45.81994117825418 1loss/FBt1/assignment_loss‚0.46663053160481327 1loss/FBt3/assignment_loss‚0.34099090937073434 0loss/SRqqt1/detection_loss‚2.890392998090157 -loss/FBt1/detection_loss‚2.60163941520911 /loss/FBt4/detection_loss‚3.8964579701423645 trainer/global_step‚7899 0loss/FRt2/assignment_loss‚3.1565333916382357 .loss/FRt2/detection_loss‚2.081067446957935 % _timestamp‚1.7884511443692672e+09 /loss/FRt1/assignment_loss‚3.764125966256665‰Ç'q³žÕ‚(:“ Ò²æÔ‹è•þ gpu.0.gpu‚42.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚141.078' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.215600000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÈX 8qÖ‚(:æ Ò²æÔÒà¤ÿ disk.sda2.out‚0 cpu‚25.337838497309725' disk./.usagePercent‚20.296843603236 disk.rbd0.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0* proc.memory.percent‚15.674853324890137 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125& disk.nvme0n1.out‚25596.92333984375# proc.memory.rssMB‚ 2568.16796875( disk.nvme0c0n1.out‚25596.92333984375 disk.rbd0.out‚0 disk.sda1.in‚0 disk.sda2.in‚0 proc.cpu.threads‚17 network.sent‚ 1.8933902e+07 disk.drbd2080.in‚0 disk.sda3.out‚ 241.109375# disk.dm-0.out‚25596.92333984375 disk.rbd1.out‚ 34.1044921875 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2265.in‚0( proc.memory.availableMB‚ 1283.9140625 network.recv‚ 2.242757e+06 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0$ memory_percent‚92.16361045837402% disk./.usageGB‚302.43763732910156 disk.sda.in‚ 0.01171875 disk.dm-1.out‚ 241.109375 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.sda.out‚ 241.109375;]P“ׂ(:ˆ á²æÔÑÆý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.191' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.6382 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¨a:wØ‚(:ì á²æÔáç„ÿ# proc.memory.rssMB‚ 2568.16796875 disk.dm-1.out‚ 241.88671875$ disk./.usageGB‚302.4430046081543 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd1.in‚0 disk.sda.in‚ 0.01171875 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0 network.recv‚ 2.248278e+06 disk.drbd2080.in‚0 disk.sda.out‚ 241.88671875 disk.drbd1716.out‚0* proc.memory.percent‚15.674853324890137 cpu‚25.337156781909655 proc.cpu.threads‚17 network.sent‚ 1.9041777e+07" disk.dm-0.out‚25649.5693359375) proc.memory.availableMB‚ 1283.56640625 disk.sda3.out‚ 241.88671875 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.drbd2265.in‚0$ memory_percent‚92.16573238372803* disk./.usagePercent‚20.297203805836677 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚25649.5693359375% disk.nvme0n1.out‚25649.5693359375 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.in‚0 disk.rbd1.out‚ 34.1044921875 disk.drbd1716.in‚0 disk.rbd0.out‚0]„åÅ2Ù- _wandbruntime‚6295  _runtime‚6295º7~)ÆÚÀ _runtime‚ 6295.83146934 _step‚650 & lr-AdamW/pg1‚4.013059329916412e-05 & lr-AdamW/pg2‚4.013059329916412e-05 trainer/global_step‚7949 % _timestamp‚1.7884511803598413e+09Ü™ùËÛÅ & lr-AdamW/pg1‚4.013059329916412e-05 & lr-AdamW/pg2‚4.013059329916412e-05 trainer/global_step‚7949 % _timestamp‚1.7884511803598413e+09 _runtime‚ 6295.83146934 _step‚650ŠÁN2Ü- _wandbruntime‚6295  _runtime‚6295·8$°ÝáO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 115519464 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 % _timestamp‚1.7884511803606067e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 265547964 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZLDeviceStatsMonitor.on}¿æÉ7#_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6630768312468 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚468600590606848 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚468599916241408 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 150008460 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚504461047004672 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚449844562426880 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6981562004992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 265527924 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6655268392960 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚461945304210944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚461945322213888 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚468599916241408 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 105395394 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414G8’Ýì'ÞæO ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚456475347778708 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚67399814 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 150008460 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚454 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6654612030464 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚67399812 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚452 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚7949 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚37996034 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚511442423542784 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚6295.832419383 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 _step‚651 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6655268392960 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 150028498 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 105395848 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 115519466 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 150028498 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚511442609009664 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚504461023064576 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚461945304210944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚461945322213888 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚468599916241408 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 105395394 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 115519464 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 % _timestamp‚1.7884511803606067e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 265547964 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6630768312468 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚468600590606848 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚468599916241408 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 150008460 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚504461047004672 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚449844562426880 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 6981562004992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 265527924 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6655268392960 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚462001657249792 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚511504971924480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6630930309676 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚67407982 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 265560394 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚462001675252736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚504522673802240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚511505136406528 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 265580460 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚432 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 105408169 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚456531068344440 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 115533482 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 150046978 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚456530396867116 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚449899483596800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38000619 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6982298122240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚7949 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6656087834112 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 115533480 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚449899466557440 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6655431458304 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 140541952 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 115533480 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 265560394 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚67407980 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚462001675252736 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 115533482 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6982438664192 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 150026914 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884511810515008e+09 _runtime‚6296.523641463 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 265580460 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚432 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 105408169 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚456531068344440 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 115533482 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 150046978 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚456530396867116 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚449899483596800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38000619 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 6982298122240 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚7949 YGDeviceStatsMonitorÎ!b_.on_train_batch_end/active_bytes.small_pool.allocated‚ 6656087834112 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 115533480 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚449899466557440 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6655431458304 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 140541952 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 115533480 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 265560394 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚67407980 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚462001675252736 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 115533482 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 6982438664192 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 150026914 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884511810515008e+09 _runtime‚6296.523641463 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚462001657249792 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚511504971924480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6630930309676 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚67407982 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 265560394 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚462001675252736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚504522673802240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚511505136406528ŒàŠj2â- _wandbruntime‚6296  _runtime‚6296Q?C\ãÖ .loss/FBt3/detection_loss‚0.695000998039223 _runtime‚6296.524601595 0loss/FRt2/assignment_loss‚3.6456926226615907 2loss/SRqqt2/assignment_loss‚1.9946721807323597 0loss/FBt2/assignment_loss‚0.5207449668510077 0loss/FBt3/assignment_loss‚0.1758589923474921 %loss/total_loss‚48.02714151109192 trainer/global_step‚7949 1loss/SRqqt3/assignment_loss‚0.952342150027535 1loss/FBt1/assignment_loss‚0.30438313931156247 /loss/FRt2/detection_loss‚3.0181466857592265 /loss/FRt4/detection_loss‚2.2721449721318026 1loss/SRqqt1/detection_loss‚2.5562601312994957 0loss/SRqqt3/detection_loss‚1.210797689800315 % _timestamp‚1.7884511810531263e+09 _step‚653 0loss/FRt1/assignment_loss‚3.7424912711848384 0loss/FRt3/assignment_loss‚2.0072329078729334 2loss/SRqqt4/assignment_loss‚1.3985270384953103 1loss/FBt4/assignment_loss‚0.25807869728721144 0loss/FRt4/assignment_loss‚3.4589102061895223 /loss/FRt1/detection_loss‚3.3041787873143735 -loss/FRt3/detection_loss‚1.70602689196284 /loss/FBt4/detection_loss‚3.1169254779815674 /loss/FBt2/detection_loss‚2.5818850614807825 epoch‚6 1loss/SRqqt2/detection_loss‚2.1402199888551556 2loss/SRqqt1/assignment_loss‚2.1824547447031364 1loss/SRqqt4/detection_loss‚1.8788703758608212 /loss/FBt1/detection_loss‚2.9052955329418184j3pbaäÛ 1loss/SRqqt2/detection_loss‚2.1402199888551556 2loss/SRqqt1/assignment_loss‚2.1824547447031364 1loss/SRqqt4/detection_loss‚1.8788703758608212 /loss/FBt1/detection_loss‚2.9052955329418184 .loss/FBt3/detection_loss‚0.695000998039223 _runtime‚6296.524601595 0loss/FRt2/assignment_loss‚3.6456926226615907 2loss/SRqqt2/assignment_loss‚1.9946721807323597 0loss/FBt2/assignment_loss‚0.5207449668510077 0loss/FBt3/assignment_loss‚0.1758589923474921 %loss/total_loss‚48.02714151109192 trainer/global_step‚7949 1loss/SRqqt3/assignment_loss‚0.952342150027535 1loss/FBt1/assignment_loss‚0.30438313931156247 /loss/FRt2/detection_loss‚3.0181466857592265 /loss/FRt4/detection_loss‚2.2721449721318026 1loss/SRqqt1/detection_loss‚2.5562601312994957 0loss/SRqqt3/detection_loss‚1.210797689800315 % _timestamp‚1.7884511810531263e+09 _step‚653 0loss/FRt1/assignment_loss‚3.7424912711848384 0loss/FRt3/assignment_loss‚2.0072329078729334 2loss/SRqqt4/assignment_loss‚1.3985270384953103 1loss/FBt4/assignment_loss‚0.25807869728721144 0loss/FRt4/assignment_loss‚3.4589102061895223 /loss/FRt1/detection_loss‚3.3041787873143735 -loss/FRt3/detection_loss‚1.70602689196284 /loss/FBt4/detection_loss‚3.1169254779815674 /loss/FBt2/detection_loss‚2.5818850614807825 epoch‚6ƒK&ž“å‚(:ˆ ð²æÔð¨–ý gpu.0.gpu‚40.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚128.941' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.7882 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0䊟sæ‚(:è ð²æÔ¸•¨ÿ proc.cpu.threads‚17 network.sent‚ 1.9048212e+07 network.recv‚ 2.253311e+06 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125) proc.memory.availableMB‚ 1283.51171875% disk.nvme0n1.out‚25695.4677734375 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0$ disk./.usageGB‚302.4099540710449 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 242.60546875 disk.sda1.out‚0 disk.rbd1.out‚ 34.13671875 disk.dm-1.out‚ 242.60546875 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.sda2.in‚0* proc.memory.percent‚15.674853324890137 disk.sda3.in‚ 0.01171875" disk.dm-0.out‚25695.4677734375) disk./.usagePercent‚20.29498575656002' disk.nvme0c0n1.out‚25695.4677734375 disk.sda3.out‚ 242.60546875 disk.sda1.in‚0$ memory_percent‚92.16606616973877# proc.memory.rssMB‚ 2568.16796875 cpu‚25.33647393425575½ÍX,“ç‚(:ˆ ÿ²æÔÀ„‹þ gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.297' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.6594 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÚþçÑvè‚(:ë ÿ²æÔܘž€ disk.dm-1.out‚ 242.98828125 proc.cpu.threads‚17 disk.rbd0.out‚0 network.sent‚ 1.9156033e+07 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda2.in‚0 disk.drbd1716.out‚0) proc.memory.availableMB‚ 1283.49609375# proc.memory.rssMB‚ 2568.16796875* proc.memory.percent‚15.674853324890137 disk.dm-0.in‚0.078125# disk.dm-0.out‚25772.85107421875) disk./.usagePercent‚20.29656532304628$ disk./.usageGB‚302.4334907531738 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 242.98828125 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚25772.85107421875 network.recv‚ 2.258585e+06 disk.rbd0.in‚0$ memory_percent‚92.16616153717041 disk.drbd2080.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 34.26171875 disk.drbd1716.in‚0 cpu‚25.33578900165372( disk.nvme0c0n1.out‚25772.85107421875 disk.sda3.out‚ 242.98828125 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875wÂážé‚(:“ Ž³æÔ៫ý gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.525' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.504999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿­Û÷nê‚(:ã Ž³æÔ『ÿ) proc.memory.availableMB‚ 1283.24609375 disk.drbd2080.out‚0 disk.rbd1.out‚ 34.41015625 disk.dm-1.in‚ 0.01171875 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚0.078125$ disk./.usageGB‚302.3587074279785 disk.drbd2266.out‚0 disk.sda1.in‚0& disk.nvme0n1.out‚25819.12841796875 disk.drbd1716.out‚0 network.sent‚ 1.9162835e+07 disk.sda3.in‚ 0.01171875 disk.sda2.in‚0# proc.memory.rssMB‚ 2568.16796875 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 243.65625 disk.dm-1.out‚ 243.65625 disk.drbd2265.in‚0$ memory_percent‚92.16768741607666 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.rbd1.in‚0 cpu‚25.33508483979733 proc.cpu.threads‚17 network.recv‚ 2.263853e+06* disk./.usagePercent‚20.291546551345036( disk.nvme0c0n1.out‚25819.12841796875 disk.rbd0.in‚0 disk.sda2.out‚0* proc.memory.percent‚15.674853324890137 disk.sda3.out‚ 243.65625# disk.dm-0.out‚25819.12841796875 disk.sda.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2265.out‚0ºc† 2ë- _wandbruntime‚6332  _runtime‚6332ð{ YÈì _step‚654 ' lr-AdamW/pg1‚3.9282226454450645e-05 ' lr-AdamW/pg2‚3.9282226454450645e-05 trainer/global_step‚7999 $ _timestamp‚1.788451217522151e+09 _runtime‚6332.993732302-F˜²ÍíÇ _runtime‚6332.993732302 _step‚654 ' lr-AdamW/pg1‚3.9282226454450645e-05 ' lr-AdamW/pg2‚3.9282226454450645e-05 trainer/global_step‚7999 $ _timestamp‚1.788451217522151e+09Ž&ú-ÿ2î- _wandbruntime‚6332  _runtime‚6332^d—ç'ïáO WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚471459228355072 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚464762956153344 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚514568175454720 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚464762974156288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7024591566848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 % _timestamp‚1.7884512175228567e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 116220264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6695597836288 _step‚655 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7024427942912 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚459260718169000 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚459261389646116 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 150934512 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6695597836288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚464762956153344 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚471458553989632 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 116220266 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 150954550 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚38225754 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚514567987890688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 116220264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚67808212 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _runtime‚6332.994697483 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 150954550 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 116220266 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚471459228355072 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 150934512 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚452589785994240 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚507543583887872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚471458553989632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚452589768954880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6671603651876 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6696254198784 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 106033968 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 106033511 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚464762974156288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 267154776 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚507543559947776 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6696254198784 trainer/global_step‚7999 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 267174816 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 116220264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6695597836288 _step‚655 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7024427942912 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚459260718169000 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚459261389646116 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 150934512 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6695597836288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚464762956153344 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚471458553989632 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 116220266 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 150954550 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚38225754 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚514567987890688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 116220264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚67808212 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _runtime‚6332.994697483 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 150954550 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 116220266 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚471459228355072 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 150934512 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚452589785994240 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚507543583887872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚471458553989632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚452589768954880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6671603651876 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6696254198784 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 106033968 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 106033511 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚464762974156288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 267154776 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚507543559947776 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6696254198784 trainer/global_step‚7999 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 267174816 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚464819309192192 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 267187246 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚507605234625536 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 trainer/global_step‚7999 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚464819309192192 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 106046240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7025298237952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6696417264128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 267207312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 150973030 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚434 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 116234282 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 116234282 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38230294 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 150952966 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6697073639936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚452644673085440 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚38229860 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚514630690182656 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚514630508923392 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚507605210685440 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚459316438734524 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 267187246 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6671765649084 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚471516400835072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6672420087048 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚471515726456320 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚67816380 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _step‚656 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 106046676 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 150952966 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7025455557120 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 150973030 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 267207312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚464819327195136 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚459317110211848 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0ysíýö&óðM D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 267187246 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6671765649084 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚471516400835072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6672420087048 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚471515726456320 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚67816380 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _step‚656 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 106046676 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 150952966 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7025455557120 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 150973030 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 267207312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚464819327195136 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚459317110211848 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6696417264128 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 116234280 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _runtime‚6333.688483195 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚464819327195136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 116234280 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚471516400835072 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚464819309192192 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 267187246 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚507605234625536 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 trainer/global_step‚7999 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚464819309192192 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 106046240 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7025298237952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6696417264128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 267207312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 150973030 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚434 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 116234282 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 116234282 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38230294 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 150952966 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6697073639936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚452644673085440 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚38229860 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚514630690182656 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚514630508923392 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚507605210685440 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚459316438734524 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414òÔ2ô- _wandbruntime‚6333  _runtime‚6333 wÂZõÔ /loss/FRt1/assignment_loss‚4.532372746202681 1loss/FBt4/assignment_loss‚0.09758237260393798 /loss/FBt2/detection_loss‚3.2962970063090324 .loss/FBt4/detection_loss‚3.502091094851494 1loss/SRqqt2/assignment_loss‚1.279667229426128 1loss/FBt2/assignment_loss‚0.49891546275443943 1loss/SRqqt2/detection_loss‚1.7917264912415434 1loss/SRqqt3/detection_loss‚1.0008328560575404 /loss/FBt3/detection_loss‚0.7891476072871418 epoch‚6 $ _timestamp‚1.788451218218185e+09 /loss/FRt3/assignment_loss‚2.284570055492854 2loss/SRqqt3/assignment_loss‚1.3047887418857391 /loss/FRt2/assignment_loss‚3.347847433585041 0loss/FRt4/assignment_loss‚2.7493367375329485 2loss/SRqqt1/assignment_loss‚1.9725149488624405 0loss/FBt1/assignment_loss‚0.5198314019610929 .loss/FRt1/detection_loss‚2.958237490720219 /loss/FRt2/detection_loss‚1.9374323993358973 /loss/FRt3/detection_loss‚1.8641160129869387 /loss/FRt4/detection_loss‚2.5207805004577306 2loss/SRqqt4/assignment_loss‚1.2094864636659621 1loss/FBt3/assignment_loss‚0.11463730943031786 0loss/SRqqt1/detection_loss‚2.487719218520557 /loss/FBt1/detection_loss‚3.4945337091173445 %loss/total_loss‚47.12190984902868 trainer/global_step‚7999 1loss/SRqqt4/detection_loss‚1.5674445587396622 _runtime‚6333.689683651 _step‚657ýî™+_öÙ /loss/FRt1/assignment_loss‚4.532372746202681 1loss/FBt4/assignment_loss‚0.09758237260393798 /loss/FBt2/detection_loss‚3.2962970063090324 .loss/FBt4/detection_loss‚3.502091094851494 1loss/SRqqt2/assignment_loss‚1.279667229426128 1loss/FBt2/assignment_loss‚0.49891546275443943 1loss/SRqqt2/detection_loss‚1.7917264912415434 1loss/SRqqt3/detection_loss‚1.0008328560575404 /loss/FBt3/detection_loss‚0.7891476072871418 epoch‚6 $ _timestamp‚1.788451218218185e+09 /loss/FRt3/assignment_loss‚2.284570055492854 2loss/SRqqt3/assignment_loss‚1.3047887418857391 /loss/FRt2/assignment_loss‚3.347847433585041 0loss/FRt4/assignment_loss‚2.7493367375329485 2loss/SRqqt1/assignment_loss‚1.9725149488624405 0loss/FBt1/assignment_loss‚0.5198314019610929 .loss/FRt1/detection_loss‚2.958237490720219 /loss/FRt2/detection_loss‚1.9374323993358973 /loss/FRt3/detection_loss‚1.8641160129869387 /loss/FRt4/detection_loss‚2.5207805004577306 2loss/SRqqt4/assignment_loss‚1.2094864636659621 1loss/FBt3/assignment_loss‚0.11463730943031786 0loss/SRqqt1/detection_loss‚2.487719218520557 /loss/FBt1/detection_loss‚3.4945337091173445 %loss/total_loss‚47.12190984902868 trainer/global_step‚7999 1loss/SRqqt4/detection_loss‚1.5674445587396622 _runtime‚6333.689683651 _step‚657‘ëŒ`‘ž÷‚(:“ ³æÔ¹Úúü gpu.0.gpu‚26.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.905' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.580999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÆÀXrø‚(:ç ³æÔ¼þ‡ÿ disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17* disk./.usagePercent‚20.299283611044842 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda1.in‚0 disk.rbd0.out‚0 disk.sda.out‚ 244.1640625 disk.sda2.out‚0 disk.dm-1.out‚ 244.1640625 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 244.1640625 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0$ memory_percent‚92.16785430908203 cpu‚25.33441440066963 network.recv‚ 2.267244e+06% disk./.usageGB‚302.47399520874023 disk.rbd0.in‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚25863.27783203125 disk.sda.in‚ 0.01171875 disk.sda1.out‚0* proc.memory.percent‚15.674853324890137 network.sent‚ 1.9168085e+07( disk.nvme0c0n1.out‚25863.27783203125 disk.rbd1.out‚ 34.41015625& disk.nvme0n1.out‚25863.27783203125 disk.drbd2265.out‚0& proc.memory.availableMB‚ 1283.21875# proc.memory.rssMB‚ 2568.16796875QVÄžù‚(:“ ¬³æÔ牮ý gpu.0.gpu‚31.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.737' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.547399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0DmhEvú‚(:ë ¬³æÔâ¢õ€ network.sent‚ 1.9277262e+07 network.recv‚ 2.274332e+06 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda2.out‚0 proc.cpu.threads‚17* disk./.usagePercent‚20.301536221338626 disk.drbd2080.out‚0 disk.drbd2266.out‚0# proc.memory.rssMB‚ 2568.16796875 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.rbd1.out‚ 34.55859375 disk.dm-1.in‚ 0.01171875& proc.memory.availableMB‚ 1282.96875 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.out‚ 244.76953125 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 disk.sda.out‚ 244.76953125* proc.memory.percent‚15.674853324890137 cpu‚25.333692281514992 disk.sda3.out‚ 244.76953125# disk.dm-0.out‚25890.07470703125 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚25890.07470703125% disk./.usageGB‚302.50756072998047( disk.nvme0c0n1.out‚25890.07470703125$ memory_percent‚92.16938018798828Øilê2û- _wandbruntime‚6370  _runtime‚6370»ÇüÁ & lr-AdamW/pg1‚3.843887380686621e-05 & lr-AdamW/pg2‚3.843887380686621e-05 trainer/global_step‚8049 % _timestamp‚1.7884512545816107e+09 _runtime‚6370.053195413 _step‚658M:×ÍÌýÆ trainer/global_step‚8049 % _timestamp‚1.7884512545816107e+09 _runtime‚6370.053195413 _step‚658 & lr-AdamW/pg1‚3.843887380686621e-05 & lr-AdamW/pg2‚3.843887380686621e-05’DðÇ52þ- _wandbruntime‚6370  _runtime‚6370fàû&ÌÿáO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚455334975482880 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚474317866103296 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 116921066 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 106673073 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚455334992522240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _runtime‚6370.054306661 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚474317191737856 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚517693606942208 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚659 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 116921064 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 268781628 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚467580626098688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6736583642112 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 trainer/global_step‚8049 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7067677929472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚455334975482880 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚474317866103296 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 116921066 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 106673073 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚455334992522240 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _runtime‚6370.054306661 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚474317191737856 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚517693606942208 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚659 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 116921064 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 268781628 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚467580626098688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6736583642112 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 trainer/global_step‚8049 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7067677929472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚454 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 151879018 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚510687771508736 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 268814098 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚467636979137536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 106685803 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚455389896652800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚474375038583296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6737403069952 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 151899082 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7068366365184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚474375038583296 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚68224782 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚462102480601932 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6713255426456 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚456 _runtime‚6370.751220127 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6738059445760 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 116935082 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚467636961134592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 268834164 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚38460567 _step‚660 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚474374364204544 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚510687747568640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 268834164 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 151899082 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 $ _timestamp‚1.788451255279159e+09 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 151879018 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 106685347 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚517756303581696 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38461021 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚462103152079256 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0Á>þ´ƒ ïM SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚462102480601932 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6713255426456 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚456 _runtime‚6370.751220127 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6738059445760 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 116935082 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚467636961134592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 268834164 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚38460567 _step‚660 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚474374364204544 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚510687747568640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 268834164 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 151899082 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 $ _timestamp‚1.788451255279159e+09 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 151879018 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 106685347 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚517756303581696 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38461021 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚462103152079256 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚467636961134592 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 116935080 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚68224780 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6737403069952 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6712600988492 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚454 TFDe¡~ûþAviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 151879018 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚510687771508736 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 268814098 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚467636979137536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 106685803 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚455389896652800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚474375038583296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6737403069952 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 151899082 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7068366365184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚474375038583296 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚68224782 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0”¿yÒ2„ - _wandbruntime‚6370  _runtime‚6370T±’êY… Ó 1loss/FBt4/assignment_loss‚0.26733077969402075 /loss/FRt1/detection_loss‚3.0428796392498594 1loss/SRqqt4/detection_loss‚1.5421697425431218 /loss/FBt2/detection_loss‚1.7906301711734973 /loss/FBt4/detection_loss‚3.2380207975705466 1loss/SRqqt3/detection_loss‚1.4835453214882701 /loss/FRt1/assignment_loss‚4.376942161357764 0loss/FBt1/assignment_loss‚0.5958565317295407 0loss/FRt3/assignment_loss‚2.1444718527793882 /loss/SRqqt1/detection_loss‚2.24387539721824 2loss/SRqqt2/assignment_loss‚1.7868363454561824 2loss/SRqqt3/assignment_loss‚1.3653220498796415 0loss/FBt2/assignment_loss‚0.5382952027414963 .loss/FRt3/detection_loss‚1.644507692232728 -loss/FRt4/detection_loss‚2.28005735874176 0loss/FRt2/assignment_loss‚3.0219090819358825 2loss/SRqqt4/assignment_loss‚1.9331508224813974 /loss/FRt2/detection_loss‚2.3408133260905744 %loss/total_loss‚45.97360393472105 % _timestamp‚1.7884512552810888e+09 1loss/SRqqt2/detection_loss‚1.6270229879480143 /loss/FBt1/detection_loss‚2.3765837875279514 /loss/FBt3/detection_loss‚0.6494044409460061 epoch‚6 trainer/global_step‚8049 _runtime‚6370.752581196 1loss/SRqqt1/assignment_loss‚2.008623023290892 _step‚661 0loss/FRt4/assignment_loss‚3.4227037715911863 0loss/FBt3/assignment_loss‚0.2526516490530897¶ ›^† Ø %loss/total_loss‚45.97360393472105 % _timestamp‚1.7884512552810888e+09 1loss/SRqqt2/detection_loss‚1.6270229879480143 /loss/FBt1/detection_loss‚2.3765837875279514 /loss/FBt3/detection_loss‚0.6494044409460061 epoch‚6 trainer/global_step‚8049 _runtime‚6370.752581196 1loss/SRqqt1/assignment_loss‚2.008623023290892 _step‚661 0loss/FRt4/assignment_loss‚3.4227037715911863 0loss/FBt3/assignment_loss‚0.2526516490530897 1loss/FBt4/assignment_loss‚0.26733077969402075 /loss/FRt1/detection_loss‚3.0428796392498594 1loss/SRqqt4/detection_loss‚1.5421697425431218 /loss/FBt2/detection_loss‚1.7906301711734973 /loss/FBt4/detection_loss‚3.2380207975705466 1loss/SRqqt3/detection_loss‚1.4835453214882701 /loss/FRt1/assignment_loss‚4.376942161357764 0loss/FBt1/assignment_loss‚0.5958565317295407 0loss/FRt3/assignment_loss‚2.1444718527793882 /loss/SRqqt1/detection_loss‚2.24387539721824 2loss/SRqqt2/assignment_loss‚1.7868363454561824 2loss/SRqqt3/assignment_loss‚1.3653220498796415 0loss/FBt2/assignment_loss‚0.5382952027414963 .loss/FRt3/detection_loss‚1.644507692232728 -loss/FRt4/detection_loss‚2.28005735874176 0loss/FRt2/assignment_loss‚3.0219090819358825 2loss/SRqqt4/assignment_loss‚1.9331508224813974 /loss/FRt2/detection_loss‚2.3408133260905744•hR1“‡ ‚(:ˆ »³æԘ߈ý gpu.0.gpu‚38.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚172.152' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.4304 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0°xKÙlˆ ‚(:á »³æÔúÊŠÿ disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875( proc.memory.availableMB‚ 1283.6953125# proc.memory.rssMB‚ 2568.16796875 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚25945.69775390625 disk.sda3.out‚ 245.21875) disk./.usagePercent‚20.30418077916733 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 network.recv‚ 2.277499e+06 disk.sda.out‚ 245.21875 disk.sda2.out‚0 disk.rbd1.out‚ 34.85546875 disk.drbd1716.in‚0* proc.memory.percent‚15.674853324890137 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 245.21875 disk.drbd2265.in‚0$ memory_percent‚92.16494560241699 proc.cpu.threads‚17& disk.nvme0n1.out‚25945.69775390625$ disk./.usageGB‚302.5469665527344 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0# disk.dm-0.out‚25945.69775390625 disk.sda2.in‚0 disk.rbd1.in‚0 cpu‚25.33304286570125 network.sent‚ 1.9282415e+07 disk.drbd2080.in‚0:ÿÜB‘‰ ‚(:† ʳæÔñùý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.31' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.062 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0*‰ú tŠ ‚(:é ʳæÔù¬¦ÿ disk.sda3.in‚ 0.01171875 disk.rbd1.in‚0 disk.nvme0c0n1.in‚0.078125% proc.memory.availableMB‚ 1284.1875 network.sent‚ 1.9391581e+07 disk.rbd1.out‚ 34.91796875 disk.dm-1.out‚ 245.91796875 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚25988.86181640625 disk.sda.in‚ 0.01171875 disk.sda3.out‚ 245.91796875 disk.rbd0.in‚0 disk.drbd1716.in‚0 cpu‚25.332379684525822 disk.drbd2265.in‚0 disk.drbd2080.out‚0* proc.memory.percent‚15.674853324890137 network.recv‚ 2.284301e+06* disk./.usagePercent‚20.305356621772095$ disk./.usageGB‚302.5644874572754# disk.dm-0.out‚25988.86181640625 disk.sda2.out‚0$ memory_percent‚92.16194152832031 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 245.91796875 disk.sda1.in‚0 disk.drbd2265.out‚0# proc.memory.rssMB‚ 2568.16796875 disk.drbd2266.out‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚25988.86181640625 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0…—·S“‹ ‚(:ˆ Ù³æÔÑ®¡ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.724' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.9448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ý1eŒ ‚(:Ú Ù³æÔ˜Á¨ÿ disk.dm-1.out‚ 246.27734375 network.sent‚ 1.939597e+07 disk.sda2.in‚0 disk.rbd1.out‚ 34.91796875 disk.drbd1716.out‚0* proc.memory.percent‚15.674853324890137 disk.sda1.out‚0 disk.sda2.out‚0) proc.memory.availableMB‚ 1284.19140625 cpu‚25.331682359012873 proc.cpu.threads‚17 network.recv‚ 2.287679e+06 disk.dm-0.out‚ 26042.90625 disk.rbd1.in‚0* disk./.usagePercent‚20.306203750702895" disk.nvme0c0n1.out‚ 26042.90625 disk.sda.out‚ 246.27734375# proc.memory.rssMB‚ 2568.16796875 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.nvme0n1.out‚ 26042.90625 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.drbd1716.in‚0# memory_percent‚92.1619176864624% disk./.usageGB‚302.57711029052734 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 246.27734375 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0Ø0} 2 - _wandbruntime‚6407  _runtime‚6407OBÔÇŽ Á % _timestamp‚1.7884512917962778e+09 _runtime‚6407.267971147 _step‚662 & lr-AdamW/pg1‚3.760071110983855e-05 & lr-AdamW/pg2‚3.760071110983855e-05 trainer/global_step‚8099 ܸÌ Æ & lr-AdamW/pg1‚3.760071110983855e-05 & lr-AdamW/pg2‚3.760071110983855e-05 trainer/global_step‚8099 % _timestamp‚1.7884512917962778e+09 _runtime‚6407.267971147 _step‚662–'Ží2 - _wandbruntime‚6407  _runtime‚6407õGÏbç'‘ áO H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚470398260038144 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚38685337 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚513708657654272 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7110534153216 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚464833473380932 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚8099 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚470398278041088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚470398278041088 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚477176503851520 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚470398260038144 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚520819167867392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚458080182010880 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 117621866 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6778225810432 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 152806654 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚460 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚464832801903816 _step‚663 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚477176503851520 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 117621864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 152786616 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚458080199050240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 270408480 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6778225810432 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6777569447936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚477175829486080 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚520819353334272 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 152786616 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6752619892936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚477175829486080 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚513708633714176 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872´L Pì'’ æO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚477175829486080 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚520819353334272 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 152786616 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6752619892936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚477175829486080 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚513708633714176 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚470398260038144 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚38685337 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚513708657654272 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7110534153216 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚464833473380932 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚8099 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚470398278041088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚470398278041088 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚477176503851520 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚470398260038144 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚520819167867392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚458080182010880 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 117621866 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6778225810432 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 152806654 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚460 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚464832801903816 _step‚663 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚477176503851520 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 117621864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 152786616 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚458080199050240 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 270408480 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6778225810432 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6777569447936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0—So§2“ - _wandbruntime‚6407  _runtime‚6407ü íª`” êM PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 107323552 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚464 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 270440950 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚2ã&Ë—$4401 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚470454631079936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6754090765864 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚477233676331520 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7111383093760 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚513770308391936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚470454613076992 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 117635880 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 152825134 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6778388875776 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚38689906 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚458135103180800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6778388875776 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚68633180 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6779045251584 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚464888522469340 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 _step‚664 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7111552995840 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 117635880 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚8099 $ _timestamp‚1.788451292502566e+09 _runtime‚6407.974755566 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 117635882 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚513770284451840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚458135086141440 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚464889193946664 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚68633182 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6753436327900 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚470454631079936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚466 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 152805070 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚477233676331520 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚477233001952768 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 107323086 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 270461016 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38690370 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 270440950 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6779045251584 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 152805070 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚520881667545600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1ôÃ]¤õ&• ïM OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚464 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 270440950 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚470454631079936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6754090765864 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚477233676331520 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7111383093760 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚513770308391936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚470454613076992 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 117635880 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 152825134 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6778388875776 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚38689906 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚458135103180800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6778388875776 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚68633180 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6779045251584 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚464888522469340 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 _step‚664 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7111552995840 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 117635880 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚8099 $ _timestamp‚1.788451292502566e+09 _runtime‚6407.974755566 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 117635882 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚513770284451840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚458135086141440 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚464889193946664 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚68633182 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6753436327900 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚470454631079936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚466 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 152805070 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚477233676331520 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚477233001952768 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 107323086 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 270461016 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38690370 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 270440950 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6779045251584 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 152805070 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚520881667545600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 107323552 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30˜Ïö¼x2– - _wandbruntime‚6407  _runtime‚6407·D)[— Õ /loss/FBt3/detection_loss‚0.6098001038846679 $ _timestamp‚1.788451292504288e+09 0loss/FRt2/assignment_loss‚2.9849733404996917 /loss/FRt4/assignment_loss‚3.215621336451117 /loss/FBt2/detection_loss‚1.8027126230299473 %loss/total_loss‚44.34016615282983 epoch‚6 trainer/global_step‚8099 2loss/SRqqt3/assignment_loss‚1.0386857318452611 1loss/SRqqt4/assignment_loss‚1.313870078489437 /loss/FRt3/detection_loss‚1.8420909496822528 1loss/SRqqt3/detection_loss‚0.9769823092760311 .loss/FBt4/detection_loss‚3.575791609287262 2loss/SRqqt1/assignment_loss‚2.1231602335141764 0loss/FBt1/assignment_loss‚0.6208322528335594 1loss/FBt4/assignment_loss‚0.12547817984595894 0loss/SRqqt1/detection_loss‚2.341157559719351 /loss/FBt1/detection_loss‚2.6264190077781677 _runtime‚6407.975786972 0loss/FRt3/assignment_loss‚2.0063122851508006 2loss/SRqqt2/assignment_loss‚1.2512381135796506 0loss/FBt2/assignment_loss‚0.5607196178752929 1loss/FBt3/assignment_loss‚0.22178578348955313 _step‚665 1loss/SRqqt2/detection_loss‚1.8997751638914149 /loss/FRt2/detection_loss‚2.4201829905917003 /loss/FRt1/assignment_loss‚4.267840969947077 /loss/FRt1/detection_loss‚2.8907061180760785 /loss/FRt4/detection_loss‚2.0820533172139584 1loss/SRqqt4/detection_loss‚1.5419764768774227äÑ-`˜ Ú /loss/FRt4/detection_loss‚2.0820533172139584 1loss/SRqqt4/detection_loss‚1.5419764768774227 /loss/FBt3/detection_loss‚0.6098001038846679 $ _timestamp‚1.788451292504288e+09 0loss/FRt2/assignment_loss‚2.9849733404996917 /loss/FRt4/assignment_loss‚3.215621336451117 /loss/FBt2/detection_loss‚1.8027126230299473 %loss/total_loss‚44.34016615282983 epoch‚6 trainer/global_step‚8099 2loss/SRqqt3/assignment_loss‚1.0386857318452611 1loss/SRqqt4/assignment_loss‚1.313870078489437 /loss/FRt3/detection_loss‚1.8420909496822528 1loss/SRqqt3/detection_loss‚0.9769823092760311 .loss/FBt4/detection_loss‚3.575791609287262 2loss/SRqqt1/assignment_loss‚2.1231602335141764 0loss/FBt1/assignment_loss‚0.6208322528335594 1loss/FBt4/assignment_loss‚0.12547817984595894 0loss/SRqqt1/detection_loss‚2.341157559719351 /loss/FBt1/detection_loss‚2.6264190077781677 _runtime‚6407.975786972 0loss/FRt3/assignment_loss‚2.0063122851508006 2loss/SRqqt2/assignment_loss‚1.2512381135796506 0loss/FBt2/assignment_loss‚0.5607196178752929 1loss/FBt3/assignment_loss‚0.22178578348955313 _step‚665 1loss/SRqqt2/detection_loss‚1.8997751638914149 /loss/FRt2/detection_loss‚2.4201829905917003 /loss/FRt1/assignment_loss‚4.267840969947077 /loss/FRt1/detection_loss‚2.8907061180760785™FÊù“™ ‚(:ˆ è³æÔ²ßéü gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.174' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.4348 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“ßøqš ‚(:æ è³æÔà„‡ÿ disk.sda.in‚ 0.01171875# disk.nvme0n1.out‚26086.12890625 disk.rbd0.out‚0 proc.cpu.threads‚17% disk./.usageGB‚302.59033584594727 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 246.88671875 cpu‚25.331029384880306* disk./.usagePercent‚20.307091328824107 disk.sda.out‚ 246.88671875 disk.sda2.out‚0 disk.drbd1716.in‚0% disk.nvme0c0n1.out‚26086.12890625 disk.drbd2266.in‚0( proc.memory.availableMB‚ 1284.1640625 network.sent‚ 1.9402438e+07 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2266.out‚0* proc.memory.percent‚15.674853324890137 network.recv‚ 2.292709e+06 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875$ memory_percent‚92.16208457946777 disk.dm-0.out‚26086.12890625 disk.sda1.out‚0 disk.dm-1.out‚ 246.88671875 disk.drbd2265.out‚0# proc.memory.rssMB‚ 2568.16796875 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.rbd1.out‚ 34.9501953125 disk.nvme0n1.in‚0.078125 disk.rbd0.in‚0îï“› ‚(:ˆ ÷³æÔÆü¥ý gpu.0.gpu‚35.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚131.464' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.2928 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0·?Ììqœ ‚(:æ ÷³æÔúýèþ disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚92.16063022613525 disk.nvme0n1.in‚0.078125# disk.dm-0.out‚26140.73193359375# proc.memory.rssMB‚ 2568.16796875 proc.cpu.threads‚17% disk./.usageGB‚302.60649490356445 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚26140.73193359375 disk.sda3.out‚ 247.40625 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 35.0751953125 network.sent‚ 1.9510423e+07* disk./.usagePercent‚20.308175776739148 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.sda.out‚ 247.40625& disk.nvme0n1.out‚26140.73193359375 disk.dm-1.out‚ 247.40625 disk.drbd1716.in‚0 disk.sda2.out‚0 network.recv‚ 2.298508e+06 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0) proc.memory.availableMB‚ 1284.40234375 cpu‚25.33034050094717 disk.drbd1716.out‚0 disk.rbd1.in‚0* proc.memory.percent‚15.674853324890137 disk.drbd2266.in‚0 disk.drbd2266.out‚0j1÷Ó2 - _wandbruntime‚6444  _runtime‚6444¸™ƒRÇž Á _step‚666 & lr-AdamW/pg1‚3.676791303521746e-05 & lr-AdamW/pg2‚3.676791303521746e-05 trainer/global_step‚8149 % _timestamp‚1.7884513286816626e+09 _runtime‚6444.153292263' ÌŸ Æ & lr-AdamW/pg1‚3.676791303521746e-05 & lr-AdamW/pg2‚3.676791303521746e-05 trainer/global_step‚8149 % _timestamp‚1.7884513286816626e+09 _runtime‚6444.153292263 _step‚666š.§ ß2  - _wandbruntime‚6444  _runtime‚6444Z¡Ls-¡ àO YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚467619515248340 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 107948056 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 trainer/global_step‚8149 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6818555253760 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚69033412 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚667 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6794109670100 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚6444.154155863 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 153712668 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 174109696 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 153712668 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7153643078144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚460825388538880 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 153732706 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚523944813675520 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚480035141599744 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7153817187840 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚473215929983488 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚516791194537472 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 272055372 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚456 $ _timestamp‚1.788451328682397e+09 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚473215911980544 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 118322664 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚480035141599744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚480034467234304 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚38915100 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚473215929983488 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚480034467234304 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚38915100 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚473215929983488 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 trainer/global_step‚8149 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6818555253760 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚69033412 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚667 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6794109670100 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚6444.154155863 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 153712668 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 174109696 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 153712668 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7153643078144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚460825388538880 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 153732706 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚523944813675520 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚480035141599744 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7153817187840 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚473215929983488 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚516791194537472 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 272055372 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚456 $ _timestamp‚1.788451328682397e+09 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚473215911980544 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 118322664 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚480035141599744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚480092314079744 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 272067802 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 272067802 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚524007342883840 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚516852845275136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6819374681600 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6820031057408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6820031057408 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚456 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 _runtime‚6445.072757819 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 107960760 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚516852821335040 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚467675235814072 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚460880309708800 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 272087868 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚69041582 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 153731122 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚480091639700992 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚467674564336748 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚460880292669440 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7154521548800 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7154676770816 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 153751186 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38919634 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 118336680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521¸"ëŽ$¥ ïM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚473272265019392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚473272283022336 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚473272283022336 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 118336682 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 107961216 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚480092314079744 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 272067802 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 272067802 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚524007342883840 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚516852845275136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6819374681600 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6820031057408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6820031057408 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚456 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 _runtime‚6445.072757819 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 107960760 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚516852821335040 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚467675235814072 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚460880309708800 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 272087868 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚69041582 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 153731122 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚480091639700992 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚467674564336748 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚460880292669440 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7154521548800 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7154676770816 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 153751186 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚38919634 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 118336680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.sma_êíÞll_pool.peak‚ 861853521 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 118336682 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 272087868 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6819374681600 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414œBXX2¦ - _wandbruntime‚6445  _runtime‚6445Aœ¯Z§ Ô 0loss/SRqqt3/detection_loss‚1.154941259390291 2loss/SRqqt1/assignment_loss‚2.3841535798434554 /loss/FRt3/detection_loss‚1.6201807136072757 /loss/FBt4/detection_loss‚3.0129849116007485 &loss/total_loss‚47.219869999479556 /loss/FBt1/detection_loss‚2.8586997509002687 _runtime‚6445.074264643 0loss/FRt2/assignment_loss‚2.6308887096551747 0loss/FRt4/assignment_loss‚2.9644144332408904 1loss/SRqqt3/assignment_loss‚1.165020565611913 -loss/FRt2/detection_loss‚2.42489516047331 1loss/SRqqt2/assignment_loss‚1.632740859069269 0loss/FBt1/assignment_loss‚0.5289055610436966 1loss/FBt2/assignment_loss‚0.31189720928668657 1loss/SRqqt1/detection_loss‚2.6289307495643355 /loss/FBt3/detection_loss‚0.8415843622032099 % _timestamp‚1.7884513296027803e+09 _step‚669 /loss/FRt1/assignment_loss‚4.336319143121893 0loss/FBt4/assignment_loss‚1.6480814417203267 1loss/SRqqt2/detection_loss‚1.8620894998311996 /loss/FBt2/detection_loss‚2.2525293588638307 1loss/FBt3/assignment_loss‚0.24652416651267628 /loss/FRt4/detection_loss‚2.1560077691078186 2loss/SRqqt4/assignment_loss‚1.5897107174936331 .loss/FRt1/detection_loss‚3.352984282103452 0loss/SRqqt4/detection_loss‚1.680325244676392 epoch‚6 trainer/global_step‚8149 0loss/FRt3/assignment_loss‚1.9350605505578062ÙÈߧ_¨ Ù epoch‚6 trainer/global_step‚8149 0loss/FRt3/assignment_loss‚1.9350605505578062 0loss/SRqqt3/detection_loss‚1.154941259390291 2loss/SRqqt1/assignment_loss‚2.3841535798434554 /loss/FRt3/detection_loss‚1.6201807136072757 /loss/FBt4/detection_loss‚3.0129849116007485 &loss/total_loss‚47.219869999479556 /loss/FBt1/detection_loss‚2.8586997509002687 _runtime‚6445.074264643 0loss/FRt2/assignment_loss‚2.6308887096551747 0loss/FRt4/assignment_loss‚2.9644144332408904 1loss/SRqqt3/assignment_loss‚1.165020565611913 -loss/FRt2/detection_loss‚2.42489516047331 1loss/SRqqt2/assignment_loss‚1.632740859069269 0loss/FBt1/assignment_loss‚0.5289055610436966 1loss/FBt2/assignment_loss‚0.31189720928668657 1loss/SRqqt1/detection_loss‚2.6289307495643355 /loss/FBt3/detection_loss‚0.8415843622032099 % _timestamp‚1.7884513296027803e+09 _step‚669 /loss/FRt1/assignment_loss‚4.336319143121893 0loss/FBt4/assignment_loss‚1.6480814417203267 1loss/SRqqt2/detection_loss‚1.8620894998311996 /loss/FBt2/detection_loss‚2.2525293588638307 1loss/FBt3/assignment_loss‚0.24652416651267628 /loss/FRt4/detection_loss‚2.1560077691078186 2loss/SRqqt4/assignment_loss‚1.5897107174936331 .loss/FRt1/detection_loss‚3.352984282103452 0loss/SRqqt4/detection_loss‚1.680325244676392ed;“© ‚(:ˆ †´æÔÙˆ›ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.307' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.2614 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0i,„ïqª ‚(:æ †´æÔ±ç–ÿ) disk./.usagePercent‚20.30877304231653 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.drbd2266.out‚0* proc.memory.percent‚15.674853324890137 proc.cpu.threads‚17 network.recv‚ 2.303877e+06 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚ 0.01171875 disk.sda.out‚ 248.015625 disk.nvme0n1.in‚0.078125 network.sent‚ 1.9517122e+07 disk.drbd1716.out‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0& disk.nvme0n1.out‚26182.01318359375 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0$ memory_percent‚92.16251373291016& proc.memory.availableMB‚ 1284.09375 cpu‚25.329688549047724# disk.dm-0.out‚26182.01318359375 disk.dm-1.in‚ 0.01171875 disk.rbd0.out‚0 disk.sda2.in‚0 disk.rbd1.out‚ 35.1376953125# proc.memory.rssMB‚ 2568.16796875% disk./.usageGB‚302.61539459228516( disk.nvme0c0n1.out‚26182.01318359375 disk.sda3.out‚ 248.015625 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.out‚ 248.015625겿“« ‚(:ˆ •´æÔˆËý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚135.403' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚27.0806 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0º¿ˆ!m¬ ‚(:â •´æÔóå€ disk.drbd1716.in‚0* proc.memory.percent‚15.674853324890137 network.sent‚ 1.9624998e+07 disk.sda3.out‚ 248.5859375 disk.drbd2266.in‚0 disk.drbd2266.out‚0! disk.dm-0.out‚26246.728515625 disk.sda.in‚ 0.01171875 disk.sda2.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 cpu‚25.32907240801743 network.recv‚ 2.309689e+06 disk.sda.out‚ 248.5859375 disk.drbd2080.out‚0% disk./.usageGB‚302.60914611816406 disk.dm-1.out‚ 248.5859375 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.sda2.in‚0 proc.cpu.threads‚17 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.in‚0& proc.memory.availableMB‚ 1284.09375$ disk.nvme0n1.out‚26246.728515625 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125& disk.nvme0c0n1.out‚26246.728515625 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 disk.sda1.in‚0 disk.rbd1.out‚ 35.1376953125$ memory_percent‚92.16251373291016# proc.memory.rssMB‚ 2568.16796875 disk.sda1.out‚0) disk./.usagePercent‚20.30835370197544©*¢­ ‚(:’ ¤´æÔßÕ¼þ gpu.0.gpu‚42.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚178.62' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.724000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0çeЧs® ‚(:è ¤´æÔÌ•ÿ disk.drbd2266.in‚0 disk.drbd2266.out‚0$ disk./.usageGB‚302.6151008605957 disk.sda.out‚ 249.11328125 disk.rbd1.in‚0 disk.drbd2265.in‚0& disk.nvme0c0n1.out‚26284.767578125 network.sent‚ 1.9631422e+07 disk.dm-1.in‚ 0.01171875 disk.sda3.out‚ 249.11328125 disk.rbd0.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125# proc.memory.rssMB‚ 2568.16796875 network.recv‚ 2.314496e+06* disk./.usagePercent‚20.308753329736394 disk.sda1.out‚0 disk.dm-0.in‚0.078125! disk.dm-0.out‚26284.767578125$ memory_percent‚92.16406345367432) proc.memory.availableMB‚ 1283.83984375$ disk.nvme0n1.out‚26284.767578125 disk.dm-1.out‚ 249.11328125 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.drbd1716.in‚0* proc.memory.percent‚15.674853324890137 cpu‚25.328400004461077 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.rbd1.out‚ 35.169921875 disk.drbd2265.out‚0 disk.rbd0.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 proc.cpu.threads‚17šmÙ¤2¯ - _wandbruntime‚6481  _runtime‚6481rÙ?QÇ° Á % _timestamp‚1.7884513656619742e+09 _runtime‚6481.133593652 _step‚670 & lr-AdamW/pg1‚3.594065313687343e-05 & lr-AdamW/pg2‚3.594065313687343e-05 trainer/global_step‚8199ç–Ð̱ Æ _runtime‚6481.133593652 _step‚670 & lr-AdamW/pg1‚3.594065313687343e-05 & lr-AdamW/pg2‚3.594065313687343e-05 trainer/global_step‚8199 % _timestamp‚1.7884513656619742e+09žeE*B2² - _wandbruntime‚6481  _runtime‚6481+L ç'³ áO E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6860197422080 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 % _timestamp‚1.7884513656627798e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6860197422080 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚69441814 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚470405557115748 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚463570612106240 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 119023464 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚39144707 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7196856860160 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚527070588280832 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚519873707480576 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚463570595066880 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚476033581925888 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚482893104982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6859541059584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚519873731420672 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚476033563922944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 154658758 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 108586521 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 119023466 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚476033563922944 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 273682224 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414O(/’ì'´ æO \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚463570595066880 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚476033581925888 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚482893104982528 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6859541059584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚519873731420672 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚476033563922944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 154658758 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 108586521 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 119023466 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚476033563922944 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 273682224 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6860197422080 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 % _timestamp‚1.7884513656627798e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6860197422080 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚69441814 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚470405557115748 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚463570612106240 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 119023464 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚39144707 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7196856860160 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚527070588280832 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚519873707480576Ÿž„@X2µ - _wandbruntime‚6481  _runtime‚6481-à © ¶ ëM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6861016863232 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7197549664256 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 119037480 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6835107006716 % _timestamp‚1.7884513663572617e+09 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 108598712 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 273694654 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚476089934964736 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚69449982 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 154677238 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.pea%#÷“Hk‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6860360487424 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚39149174 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚482950951827968 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 119037482 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚476089916961792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6861016863232 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 trainer/global_step‚8199 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 108599156 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚6481.829254191 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚69449980 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚527132907882496 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7197717469184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚470461277681480 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 154657174 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6860360487424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6835761444680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚519935358218240 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 273714720 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚527133099627520 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚470460606204156 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚463625516236800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 154677238 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚476089934964736 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚482950951827968 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 273714720 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _step‚672 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 119037482 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 273714720 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 _step‚672 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 119037482 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 108598712 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 273694654 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚476089934964736 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚69449982 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 154677238 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6860360487424 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚39149174 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚482950951827968 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 119037482 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚476089916961792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6861016863232 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 trainer/global_step‚8199 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 108599156 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚6481.829254191 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚69449980 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚527132907882496 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7197717469184 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚470461277681480 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 154657174 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6860360487424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6835761444680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚519935358218240 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 273714720 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚527133099627520 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚470460606204156 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚463625516236800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 154677238 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚476089934964736 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚482950951827968 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 p &2¸ - _wandbruntime‚6481  _runtime‚6481Lã5)T¹ Î -loss/FBt4/detection_loss‚4.41338649392128 0loss/FBt4/assignment_loss‚0.2099928930401802 trainer/global_step‚8199 1loss/SRqqt1/assignment_loss‚2.155911353826523 2loss/SRqqt4/assignment_loss‚1.6197862894332502 1loss/SRqqt3/detection_loss‚1.2877111775370744 1loss/SRqqt4/detection_loss‚1.5320654197130352 /loss/FBt1/detection_loss‚2.0377556959788006 0loss/FRt3/assignment_loss‚1.7331149211296668 epoch‚6 2loss/SRqqt2/assignment_loss‚1.4214785067737103 .loss/FRt4/detection_loss‚2.255592964449019 _runtime‚6481.830395615 _step‚673 0loss/FBt2/assignment_loss‚0.6372213036588886 /loss/FRt1/assignment_loss‚4.759698812778179 0loss/FRt2/assignment_loss‚3.5295298186441264 2loss/SRqqt3/assignment_loss‚1.1296366355023706 /loss/FRt2/detection_loss‚2.2490348666906357 .loss/FBt2/detection_loss‚2.017046844258028 0loss/FBt1/assignment_loss‚0.5064498570876216 0loss/SRqqt1/detection_loss‚2.836843273639679 /loss/FBt3/detection_loss‚0.6617276707101006 $loss/total_loss‚46.7031610164605 $ _timestamp‚1.788451366358919e+09 0loss/FRt4/assignment_loss‚2.5941116056352294 0loss/FBt3/assignment_loss‚0.2747902667055904 .loss/FRt1/detection_loss‚2.997022807598114 .loss/FRt3/detection_loss‚2.005497275063625 1loss/SRqqt2/detection_loss‚1.8377542626857757B‹LqYº Ó /loss/FBt1/detection_loss‚2.0377556959788006 0loss/FRt3/assignment_loss‚1.7331149211296668 epoch‚6 2loss/SRqqt2/assignment_loss‚1.4214785067737103 .loss/FRt4/detection_loss‚2.255592964449019 _runtime‚6481.830395615 _step‚673 0loss/FBt2/assignment_loss‚0.6372213036588886 /loss/FRt1/assignment_loss‚4.759698812778179 0loss/FRt2/assignment_loss‚3.5295298186441264 2loss/SRqqt3/assignment_loss‚1.1296366355023706 /loss/FRt2/detection_loss‚2.2490348666906357 .loss/FBt2/detection_loss‚2.017046844258028 0loss/FBt1/assignment_loss‚0.5064498570876216 0loss/SRqqt1/detection_loss‚2.836843273639679 /loss/FBt3/detection_loss‚0.6617276707101006 $loss/total_loss‚46.7031610164605 $ _timestamp‚1.788451366358919e+09 0loss/FRt4/assignment_loss‚2.5941116056352294 0loss/FBt3/assignment_loss‚0.2747902667055904 .loss/FRt1/detection_loss‚2.997022807598114 .loss/FRt3/detection_loss‚2.005497275063625 1loss/SRqqt2/detection_loss‚1.8377542626857757 -loss/FBt4/detection_loss‚4.41338649392128 0loss/FBt4/assignment_loss‚0.2099928930401802 trainer/global_step‚8199 1loss/SRqqt1/assignment_loss‚2.155911353826523 2loss/SRqqt4/assignment_loss‚1.6197862894332502 1loss/SRqqt3/detection_loss‚1.2877111775370744 1loss/SRqqt4/detection_loss‚1.5320654197130352¡8Öç|ž» ‚(:“ ³´æÔØüŽý gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚134.732' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.946399999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Žîç(r¼ ‚(:ç ³´æÔÄ–©ÿ) disk./.usagePercent‚20.30830429252134 disk.drbd2266.in‚0 disk.sda.out‚ 249.5546875& disk.nvme0n1.out‚26326.13037109375) proc.memory.availableMB‚ 1284.05859375 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0$ memory_percent‚92.16272830963135 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚26326.13037109375# disk.dm-0.out‚26326.13037109375 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.dm-1.out‚ 249.5546875 network.sent‚ 1.9636657e+07 disk.sda3.out‚ 249.5546875 disk.sda1.in‚0 disk.drbd2265.in‚0* proc.memory.percent‚15.674853324890137 network.recv‚ 2.317776e+06 disk.rbd0.in‚0 disk.rbd0.out‚0# proc.memory.rssMB‚ 2568.16796875 cpu‚25.327758289277 disk.drbd2080.in‚0 disk.rbd1.in‚0$ disk./.usageGB‚302.6084098815918 disk.nvme0c0n1.in‚0.078125 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.rbd1.out‚ 35.294921875 disk.dm-1.in‚ 0.01171875 proc.cpu.threads‚17ç ó‘½ ‚(:† ´æÔ° Šý gpu.0.gpu‚35.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚173.51' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.702 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0÷û´ëx¾ ‚(:í ´æÔ§–ªÿ disk.rbd0.out‚0 disk.dm-0.in‚0.078125* proc.memory.percent‚15.674853324890137 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 35.357421875& disk.nvme0n1.out‚26382.96337890625 disk.drbd2265.out‚0$ memory_percent‚91.97049140930176( proc.memory.availableMB‚ 1315.5546875$ disk./.usageGB‚302.6236915588379 disk.sda.out‚ 250.00390625 disk.sda2.in‚0 disk.sda2.out‚0 disk.sda3.out‚ 250.00390625 disk.drbd2266.in‚0 disk.rbd0.in‚0# disk.dm-0.out‚26382.96337890625 network.sent‚ 1.9745953e+07 disk.drbd2080.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 250.00390625# proc.memory.rssMB‚ 2568.16796875 cpu‚25.327121090129616 disk.sda.in‚ 0.01171875 proc.cpu.threads‚17 network.recv‚ 2.325175e+06* disk./.usagePercent‚20.309329858703503 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.drbd1716.out‚0 disk.dm-1.in‚ 0.01171875( disk.nvme0c0n1.out‚26382.96337890625 disk.sda3.in‚ 0.011718751ÐfÒ2¿ - _wandbruntime‚6519  _runtime‚6519ÔÏÿÎÉÀ à _runtime‚6519.475317332 _step‚674 ' lr-AdamW/pg1‚3.5119103814529375e-05 ' lr-AdamW/pg2‚3.5119103814529375e-05 trainer/global_step‚8249 % _timestamp‚1.7884514040036635e+09ôÿ%œÎÁ È _runtime‚6519.475317332 _step‚674 ' lr-AdamW/pg1‚3.5119103814529375e-05 ' lr-AdamW/pg2‚3.5119103814529375e-05 trainer/global_step‚8249 % _timestamp‚1.7884514040036635e+09¢B=÷Ñ2 - _wandbruntime‚6519  _runtime‚6519¦DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 119724264 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 155564772 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚69850212 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 119724264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚485752417096192 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚478851233868288 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚530196198137344 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚522956268303872 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6875125911160 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 178304000 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚473190927506040 trainer/global_step‚8249 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 109223890 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6900526865408 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 119724266 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚485751742730752 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚466315801594880 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 155564772 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 155584810 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 119724266 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚485751742730752 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 202244096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 $ _timestamp‚1.788451404004435e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚485752417096192 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 109224363 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚473 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚39374149 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 119724264 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 155564772 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚69850212 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 119724264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚485752417096192 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚478851233868288 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚530196198137344 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚522956268303872 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6875125911160 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 178304000 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚473190927506040 trainer/global_step‚8249 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 109223890 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6900526865408 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 119724266 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚485751742730752 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚466315801594880 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 155564772 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 155584810 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 119724266 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚485751742730752 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 202244096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 $ _timestamp‚1.788451404004435e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚485752417096192 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 109224363 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚473 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚39374149 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 6520.16933705 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7240773673472 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6901346293248 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 275321506 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6876596784088 _step‚676 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚473247319548888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚485808915197440 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 155583226 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚523017919041536 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 119738280 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚530258692715008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚478907586907136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7240605868544 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚8249 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 109236560 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 275341572 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚473246648071564 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 109237035 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0­5’?õ&Ç ïM OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 155603290 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚523017895101440 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6902002669056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6875942346124 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚ 6520.16933705 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7240773673472 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6901346293248 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 275321506 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6876596784088 _step‚676 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚473247319548888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚485808915197440 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 155583226 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚523017919041536 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 119738280 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚530258692715008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚478907586907136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7240605868544 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚8249 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 109236560 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 275341572 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚473246648071564 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 109237035 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0¤KŒ‚d2È - _wandbruntime‚6520  _runtime‚6520ͱÌ^]É × epoch‚6 /loss/FRt1/detection_loss‚3.0262196637116947 0loss/SRqqt2/detection_loss‚1.885025040174906 /loss/FBt1/detection_loss‚2.5785483609546316 .loss/FBt4/detection_loss‚2.783800446987152 _runtime‚6520.170856227 _step‚677 2loss/SRqqt4/assignment_loss‚1.5834386198491346 1loss/SRqqt3/detection_loss‚1.4386018323317422 /loss/FBt2/detection_loss‚2.2266035974025726 2loss/SRqqt3/assignment_loss‚1.1526577356508223 /loss/SRqqt1/detection_loss‚2.39106894493103 &loss/total_loss‚44.360306317374366 0loss/FRt3/assignment_loss‚1.6627234065974201 0loss/FRt4/assignment_loss‚3.1170841104844036 2loss/SRqqt1/assignment_loss‚1.9095877426862717 2loss/SRqqt2/assignment_loss‚1.0797362258801093 1loss/FBt1/assignment_loss‚0.42012398274916696 trainer/global_step‚8249 0loss/FRt2/assignment_loss‚3.6958653254386706 0loss/FBt4/assignment_loss‚0.5195604965090752 /loss/FRt2/detection_loss‚2.2762260230687947 /loss/FBt3/detection_loss‚0.6830887220298442 1loss/FBt2/assignment_loss‚0.28865430502355194 /loss/FRt3/detection_loss‚1.6908712533682033 .loss/FRt4/detection_loss‚2.072251398043305 1loss/SRqqt4/detection_loss‚1.4870070109113318 % _timestamp‚1.7884514046993353e+09 /loss/FRt1/assignment_loss‚4.235239349878752 1loss/FBt3/assignment_loss‚0.156322722711777635•²,¾Ê Ü 1loss/FBt2/assignment_loss‚0.28865430502355194 /loss/FRt3/detection_loss‚1.6908712533682033 .loss/FRt4/detection_loss‚2.072251398043305 1loss/SRqqt4/detection_loss‚1.4870070109113318 % _timestamp‚1.7884514046993353e+09 /loss/FRt1/assignment_loss‚4.235239349878752 1loss/FBt3/assignment_loss‚0.15632272271177763 epoch‚6 /loss/FRt1/detection_loss‚3.0262196637116947 0loss/SRqqt2/detection_loss‚1.885025040174906 /loss/FBt1/detection_loss‚2.5785483609546316 .loss/FBt4/detection_loss‚2.783800446987152 _runtime‚6520.170856227 _step‚677 2loss/SRqqt4/assignment_loss‚1.5834386198491346 1loss/SRqqt3/detection_loss‚1.4386018323317422 /³þ¤loss/FBt2/detection_loss‚2.2266035974025726 2loss/SRqqt3/assignment_loss‚1.1526577356508223 /loss/SRqqt1/detection_loss‚2.39106894493103 &loss/total_loss‚44.360306317374366 0loss/FRt3/assignment_loss‚1.6627234065974201 0loss/FRt4/assignment_loss‚3.1170841104844036 2loss/SRqqt1/assignment_loss‚1.9095877426862717 2loss/SRqqt2/assignment_loss‚1.0797362258801093 1loss/FBt1/assignment_loss‚0.42012398274916696 trainer/global_step‚8249 0loss/FRt2/assignment_loss‚3.6958653254386706 0loss/FBt4/assignment_loss‚0.5195604965090752 /loss/FRt2/detection_loss‚2.2762260230687947 /loss/FBt3/detection_loss‚0.6830887220298442¥u¢éd“Ë ‚(:ˆ Ñ´æÔÔȧý gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.804' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.9608 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0’‹ÊŽgÌ ‚(:Ü Ñ´æÔò†²ÿ) proc.memory.availableMB‚ 1315.74609375 cpu‚25.326486348025664 network.sent‚ 1.975117e+07 disk.sda2.out‚0 disk.rbd1.out‚ 35.357421875 disk.rbd0.in‚0 proc.cpu.threads‚17 disk.sda2.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125$ memory_percent‚91.96932315826416# proc.memory.rssMB‚ 2568.16796875* proc.memory.percent‚15.674853324890137 disk.sda.in‚ 0.01171875 disk.sda.out‚250.625& disk.nvme0n1.out‚26418.99658203125 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125* disk./.usagePercent‚20.310026711211705 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0( disk.nvme0c0n1.out‚26418.99658203125 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0$ disk./.usageGB‚302.6340751647949 disk.dm-1.out‚250.625 disk.drbd1716.out‚0 disk.rbd0.out‚0# disk.dm-0.out‚26418.99658203125 network.recv‚ 2.3285e+06 disk.sda3.out‚250.625rÀB“Í ‚(:ˆ à´æÔê§ý gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.229' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4458 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0pâUQjÎ ‚(:ß à´æÔ×Ȥÿ# proc.memory.rssMB‚ 2568.16796875 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.rbd1.out‚ 35.3896484375 disk.sda.out‚ 251.265625 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.dm-1.out‚ 251.265625 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 251.265625" proc.memory.availableMB‚1315.5 network.sent‚ 1.986053e+07# disk.dm-0.out‚26453.18408203125 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2266.out‚0# memory_percent‚91.9708251953125) disk./.usagePercent‚20.31093451392812$ disk./.usageGB‚302.6476020812988 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 proc.cpu.threads‚17 disk.rbd1.in‚0 disk.sda.in‚ 0.01171875 disk.sda1.out‚0( disk.nvme0c0n1.out‚26453.18408203125* proc.memory.percent‚15.674853324890137 cpu‚25.325817839624925& disk.nvme0n1.out‚26453.18408203125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 network.recv‚ 2.335714e+06 disk.sda2.in‚0 disk.drbd2265.out‚0«§:r“Ï ‚(:ˆ ï´æÔ›š’ý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.229' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4458 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0bzdmÐ ‚(:â ï´æÔ©²¦ÿ) disk./.usagePercent‚20.31127551596373 disk.sda.in‚ 0.01171875 disk.drbd1716.out‚0# proc.memory.rssMB‚ 2568.16796875 network.recv‚ 2.338963e+06 disk.rbd0.out‚0 disk.sda.out‚ 251.6328125 disk.sda1.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 35.5146484375* proc.memory.percent‚15.674853324890137 network.sent‚ 1.986566e+07 disk.drbd2266.in‚0 disk.drbd2266.out‚0& disk.nvme0n1.out‚26498.11572265625 disk.drbd2265.out‚0# memory_percent‚91.9708251953125 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 251.6328125 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 251.6328125% disk./.usageGB‚302.65268325805664( disk.nvme0c0n1.out‚26498.11572265625" proc.memory.availableMB‚1315.5 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 cpu‚25.32518935423932 disk.rbd0.in‚0# disk.dm-0.out‚26498.11572265625 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2265.in‚0D:d™2Ñ - _wandbruntime‚6556  _runtime‚6556š^8ÙÇÒ Á trainer/global_step‚8299 % _timestamp‚1.7884514412214515e+09 _runtime‚6556.693084787 _step‚678 & lr-AdamW/pg1‚3.430343627783273e-05 & lr-AdamW/pg2‚3.430343627783273e-05擈ÌÓ Æ & lr-AdamW/pg2‚3.430343627783273e-05 trainer/global_step‚8299 % _timestamp‚1.7884514412214515e+09 _runtime‚6556.693084787 _step‚678 & lr-AdamW/pg1‚3.430343627783273e-05¦Ø£ÏF2Ô - _wandbruntime‚6556  _runtime‚6556áGëŽç'Õ áO OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚39603653 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚475977640850564 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 156490824 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 120425064 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 156490824 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 276915888 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 120425066 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 trainer/global_step‚8299 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚488610380478976 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚488610380478976 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚481668867807744 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚526038781246976 _step‚679 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚469061025162240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚469061008122880 % _timestamp‚1.7884514412222207e+09 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚526038805187072 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 156510862 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6942169033728 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 276935928 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚481668867807744 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚488611054844416 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚488611054844416 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚70258612 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6915961250568 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 120425064 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚533321783409152 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚481668885810688 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7282978222080 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6941512671232 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6941512671232 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚70258614 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚533321595845120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 120425066 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 109861810 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚6556.694156349 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚39603198 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚488610380478976 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚481668867807744 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚526038781246976 _step‚679 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚469061025162240 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚469061008122880 % _timestamp‚1.7884514412222207e+09 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚526038805187072 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 156510862 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6942169033728 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 276935928 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚481668867807744 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚488611054844416 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚488611054844416 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚70258612 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6915961250568 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 120425064 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚533321783409152 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚481668885810688 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7282978222080§|ß62× - _wandbruntime‚6557  _runtime‚6557ø<ª÷Ø ëM H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 276968424 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 109874618 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚70266782 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚488668227324416 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚533384292238848 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6916777685532 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 120439082 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6942332099072 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚6557.392363344 _step‚680 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 120439082 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚469115912253440 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚469115929292800 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚533384104688128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7283836314112 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚488668227324416 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚481725238849536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6942988474880 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚467 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 156509278 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 156529342 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚476033361416296 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚39608303 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 276948358 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚465 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 156529342 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚481725220846592 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚488667552945664 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 trainer/global_step‚8299 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 120439080 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 109875085 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚39607838 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 276968424 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 120439080 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚476032689938972 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚526100431984640Ë}­ö&Ù ðM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚39608303 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 276948358 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚465 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 156529342 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚481725220846592 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚488667552945664 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 trainer/global_step‚8299 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 120439080 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 109875085 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚39607838 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 276968424 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 120439080 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚476032689938972 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚526100431984640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 276968424 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 109874618 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚70266782 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚488668227324416 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚533384292238848 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6916777685532 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 120439082 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6942332099072 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚6557.392363344 _step‚680 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 120439082 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚469115912253440 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚469115929292800 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚533384104688128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7283836314112 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚488668227324416 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚481725238849536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6942988474880 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚467 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 156509278 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 156529342 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚476033361416296 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952¨þ+z`2Ú - _wandbruntime‚6557  _runtime‚6557šÙFZÛ Ô .loss/FRt2/assignment_loss‚3.69488917744678 0loss/FRt3/assignment_loss‚1.8944151376156098 .loss/FRt4/detection_loss‚2.011867663469808 /loss/FBt4/detection_loss‚3.4364928781986235 0loss/SRqqt2/detection_loss‚1.801661310603424 /loss/FBt2/detection_loss‚2.5704087064816403 &loss/total_loss‚44.414284041986804 _step‚681 0loss/FRt4/assignment_loss‚2.6376578150124383 2loss/SRqqt1/assignment_loss‚1.9645682244465268 1loss/FBt4/assignment_loss‚0.43297055587172506 /loss/FBt1/detection_loss‚2.4735071238349464 $ _timestamp‚1.788451441921936e+09 0loss/FRt1/assignment_loss‚3.6369290460239756 2loss/SRqqt4/assignment_loss‚1.1822660086819758 1loss/SRqqt3/detection_loss‚1.0174149576988485 2loss/SRqqt2/assignment_loss‚1.2841758154028533 /loss/FRt3/detection_loss‚1.6804736772591764 0loss/SRqqt1/detection_loss‚2.788843913324948 2loss/SRqqt3/assignment_loss‚1.2002108765559063 /loss/FBt3/detection_loss‚0.6996339666376631 epoch‚6 _runtime‚6557.393482296 .loss/FRt1/detection_loss‚3.372718568552624 0loss/FBt1/assignment_loss‚0.4265897785920542 0loss/FBt2/assignment_loss‚0.4193872611615986 1loss/FBt3/assignment_loss‚0.23640413839899505 .loss/FRt2/detection_loss‚2.195345568916072 1loss/SRqqt4/detection_loss‚1.3554518717985886 trainer/global_step‚8299 Ì _Ü Ù 0loss/FBt1/assignment_loss‚0.4265897785920542 0loss/FBt2/assignment_loss‚0.4193872611615986 1loss/FBt3/assignment_loss‚0.23640413839899505 .loss/FRt2/detection_loss‚2.195345568916072 1loss/SRqqt4/detection_loss‚1.3554518717985886 trainer/global_step‚8299 .loss/FRt2/assignment_loss‚3.69488917744678 0loss/FRt3/assignment_loss‚1.8944151376156098 .loss/FRt4/detection_loss‚2.011867663469808 /loss/FBt4/detection_loss‚3.4364928781986235 0loss/SRqqt2/detection_loss‚1.801661310603424 /loss/FBt2/detection_loss‚2.5704087064816403 &loss/total_loss‚44.414284041986804 _step‚681 0loss/FRt4/assignment_loss‚2.6376578150124383 2loss/SRqqt1/assignment_loss‚1.9645682244465268 1loss/FBt4/assignment_loss‚0.43297055587172506 /loss/FBt1/detection_loss‚2.4735071238349464 $ _timestamp‚1.788451441921936e+09 0loss/FRt1/assignment_loss‚3.6369290460239756 2loss/SRqqt4/assignment_loss‚1.1822660086819758 1loss/SRqqt3/detection_loss‚1.0174149576988485 2loss/SRqqt2/assignment_loss‚1.2841758154028533 /loss/FRt3/detection_loss‚1.6804736772591764 0loss/SRqqt1/detection_loss‚2.788843913324948 2loss/SRqqt3/assignment_loss‚1.2002108765559063 /loss/FBt3/detection_loss‚0.6996339666376631 epoch‚6 _runtime‚6557.393482296 .loss/FRt1/detection_loss‚3.372718568552624©gØÛ^žÝ ‚(:“ þ´æÔ³ž¡ý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.525' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.504999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÂÏsòuÞ ‚(:ê þ´æÔ©Œ«ÿ$ disk./.usageGB‚302.6614761352539 disk.nvme0n1.in‚0.078125 disk.sda1.out‚0 disk.sda2.out‚0$ memory_percent‚91.97101593017578& proc.memory.availableMB‚ 1315.46875 proc.cpu.threads‚17 network.sent‚ 1.987239e+07 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 252.44140625 disk.drbd2265.out‚0 disk.sda.out‚ 252.44140625 disk.sda2.in‚0 disk.rbd1.out‚ 35.5771484375 disk.drbd1716.in‚0( disk.nvme0c0n1.out‚26528.19580078125 disk.sda3.out‚ 252.44140625 disk.rbd0.out‚0& disk.nvme0n1.out‚26528.19580078125 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875# disk.dm-0.out‚26528.19580078125 cpu‚25.32456358211367 network.recv‚ 2.344423e+06 disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0* disk./.usagePercent‚20.311865613330152 disk.drbd2080.in‚0 disk.sda1.in‚0# proc.memory.rssMB‚ 2568.16796875 disk.drbd1716.out‚0 disk.sda.in‚ 0.01171875 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125* proc.memory.percent‚15.674853324890137Ñ»j®“ß ‚(:ˆ µæÔ‹îŸý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚127.441' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.4882 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0œz'ƒmà ‚(:â µæÔê˲ÿ disk.rbd1.out‚ 35.5771484375 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875 cpu‚25.323978253030763 disk.sda2.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 253.1171875 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.dm-0.out‚26574.37109375* proc.memory.percent‚15.674853324890137$ disk./.usageGB‚302.6652183532715 disk.sda2.in‚0# disk.nvme0n1.out‚26574.37109375 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125* disk./.usagePercent‚20.312116756721245% disk.nvme0c0n1.out‚26574.37109375 disk.rbd0.in‚0 disk.sda1.out‚0$ memory_percent‚91.97337627410889 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 253.1171875 disk.sda.in‚ 0.01171875# proc.memory.rssMB‚ 2568.16796875 proc.cpu.threads‚17 disk.sda.out‚ 253.1171875 disk.sda1.in‚0) proc.memory.availableMB‚ 1315.08203125 network.recv‚ 2.34962e+06 disk.rbd0.out‚0 network.sent‚ 1.9980156e+07Ó>‹+2á - _wandbruntime‚6593  _runtime‚6593Ùí¯-Èâ  _runtime‚ 6593.99112137 _step‚682 ' lr-AdamW/pg1‚3.3493820510675516e-05 ' lr-AdamW/pg2‚3.3493820510675516e-05 trainer/global_step‚8349 % _timestamp‚1.7884514785194805e+09ÓÅ›FÍã Ç _step‚682 ' lr-AdamW/pg1‚3.3493820510675516e-05 ' lr-AdamW/pg2‚3.3493820510675516e-05 trainer/global_step‚8349 % _timestamp‚1.7884514785194805e+09 _runtime‚ 6593.99112137ªO§ ô2ä - _wandbruntime‚6593  _runtime‚6593ªyò¢ç'å áO RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 110500406 _runtime‚6593.992106269 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚471806231690240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚484486537753088 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 121125864 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚491469692592640 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 157436914 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚491469018227200 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚484486519750144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6982498477056 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 278562780 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6983154839552 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 121125866 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6982498477056 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 110499953 trainer/global_step‚8349 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚484486519750144 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚471806214650880 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6956796589976 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚484486537753088 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 121125864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7326027440640 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 157416876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚529121342070272 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚39833392 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7325868011008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚478763682717972 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 157416876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚451 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚529121318130176 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚478763011240856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 121125866 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6957451027732 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚70667014 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚70667012 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚536447369510912 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚536447186141184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚683 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6983154839552 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚39832941 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884514785202215e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚70667014 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚70667012 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚536447369510912 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚2412ÍGD8 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚536447186141184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚683 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 6983154839552 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚39832941 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884514785202215e+09 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚491469018227200 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚484486519750144 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 6982498477056 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 278562780 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 6983154839552 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 121125866 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 6982498477056 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 110499953 trainer/global_step‚8349 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚484486519750144 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚471806214650880 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6956796589976 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚484486537753088 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 121125864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7326027440640 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 157416876 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚529121342070272« ã™Á2ç - _wandbruntime‚6594  _runtime‚6594ï Ý’ñ&è ëM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚484542890791936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚536509895206400 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6983974280704 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚484542872788992 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 157455394 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6983317904896 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚529182992807936 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 121139882 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 278575210 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 110512660 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 278595276 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7326747176448 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 157455394 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 157435330 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚491526865072640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 157435330 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6983317904896 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 278595276 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 110513114 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7884514792030993e+09 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 121139880 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚39837480 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚491526865072640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚484542890791936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚70675180 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6957613024940 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6983974280704 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚536509716044288 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088€k‚jö&é ðM L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 278595276 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7326747176448 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 157455394 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 157435330 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚491526865072640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 157435330 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 6983317904896 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 278595276 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 110513114 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7884514792030993e+09 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 121139880 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚39837480 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚491526865072640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚484542890791936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚70675180 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6957613024940 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 6983974280704 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚536509716044288 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚484542890791936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚536509895206400 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 6983974280704 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚484542872788992 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 157455394 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 6983317904896 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚529182992807936 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 121139882 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 278575210 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 110512660 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 121826664 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚474551421178880 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 121826666 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 158362966 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚532203855013376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6997631929384 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7023484282880 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚487304189695488 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 158362966 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚40062582 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚71075412 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚40062132 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6998286367140 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚494327655975424 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 280189632 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚481549724585380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 111137544 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚539573035193856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 121826664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7024140645376 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _step‚687 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7023484282880 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 121826666 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944&Ù#_ì'ø æO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚487304189695488 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚494327655975424 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7024140645376 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7368986324992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 _runtime‚6630.977266887 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 121826664 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚474551421178880 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 121826666 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 158362966 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚532203855013376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 6997631929384 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7023484282880 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚487304189695488 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 158362966 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚40062582 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚71075412 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚40062132 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 6998286367140 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚494327655975424 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 280189632 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚481549724585380 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 111137544 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚539573035193856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 121826664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7024140645376 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 _step‚687 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7023484282880 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 121826666 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 _runtime‚6631.671039237 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚539635361381888 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 111150195 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 158361382 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 111150650 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚532265505751040 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6998448364348 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚494385502820864 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚40066615 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚539635544738304 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚474606325309440 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚494384828442112 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 158381446 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚487360524731392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚474606342348800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7369855630848 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6999102802312 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 158361382 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 280222128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 158381446 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚481605445151112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚688 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884515161988297e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 280222128 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 280202062 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928”—ºcö&û ðM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚487360524731392 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚487360542734336 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 _runtime‚6631.671039237 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚539635361381888 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 111150195 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 158361382 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 111150650 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚532265505751040 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 6998448364348 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚494385502820864 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚40066615 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚539635544738304 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚474606325309440 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚494384828442112 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 158381446 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚487360524731392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚474606342348800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7369855630848 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 6999102802312 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 158361382 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 280222128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 158381446 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚481605445151112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚688 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884515161988297e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 280222128 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 280202062 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 121840682°ò_é2ü - _wandbruntime‚6631  _runtime‚66314…Xý Ò /loss/FRt1/detection_loss‚3.0977708873925387 epoch‚6 1loss/FBt3/assignment_loss‚0.21963915908537182 1loss/SRqqt4/detection_loss‚1.4567922301969285 /loss/FBt3/detection_loss‚0.7943780906633406 % _timestamp‚1.7884515162004774e+09 /loss/FRt2/assignment_loss‚3.655528353883865 0loss/SRqqt1/detection_loss‚2.846204538236965 1loss/SRqqt3/detection_loss‚0.9897682486334816 #loss/total_loss‚44.530741874464 trainer/global_step‚8399 2loss/SRqqt2/assignment_loss‚1.2704231659322978 2loss/SRqqt4/assignment_loss‚1.4081297300408704 1loss/FBt1/assignment_loss‚0.35121848441586473 /loss/FRt3/detection_loss‚1.9611105037885799 .loss/FRt4/detection_loss‚2.045726119840549 /loss/FBt1/detection_loss‚2.3964554156575884 0loss/FRt4/assignment_loss‚2.8508989194692194 2loss/SRqqt1/assignment_loss‚2.5370813526890497 1loss/FBt4/assignment_loss‚0.07686166709754616 .loss/FRt2/detection_loss‚2.006624052499203 .loss/FBt2/detection_loss‚1.990352988243103 0loss/SRqqt2/detection_loss‚1.855781337618828 _runtime‚6631.672008696 _step‚689 0loss/FRt3/assignment_loss‚1.9912309834831639 1loss/SRqqt3/assignment_loss‚0.953825729462551 0loss/FBt2/assignment_loss‚0.5579680401299676 /loss/FBt4/detection_loss‚3.1498259544372558 /loss/FRt1/assignment_loss‚4.067145921565868–¹à)]þ × 0loss/FRt4/assignment_loss‚2.8508989194692194 2loss/SRqqt1/assignment_loss‚2.5370813526890497 1loss/FBt4/assignment_loss‚0.07686166709754616 .loss/FRt2/detection_loss‚2.006624052499203 .loss/FBt2/detection_loss‚1.990352988243103 0loss/SRqqt2/detection_loss‚1.855781337618828 _runtime‚6631.672008696 _step‚689 0loss/FRt3/assignment_loss‚1.9912309834831639 1loss/SRqqt3/assignment_loss‚0.953825729462551 0loss/FBt2/assignment_loss‚0.5579680401299676 /loss/FBt4/detection_loss‚3.1498259544372558 /loss/FRt1/assignment_loss‚4.067145921565868 /loss/FRt1/detection_loss‚3.0977708873925387 epoch‚6 1loss/FBt3/assignment_loss‚0.21963915908537182 1loss/SRqqt4/detection_loss‚1.4567922301969285 /loss/FBt3/detection_loss‚0.7943780906633406 % _timestamp‚1.7884515162004774e+09 /loss/FRt2/assignment_loss‚3.655528353883865 0loss/SRqqt1/detection_loss‚2.846204538236965 1loss/SRqqt3/detection_loss‚0.9897682486334816 #loss/total_loss‚44.530741874464 trainer/global_step‚8399 2loss/SRqqt2/assignment_loss‚1.2704231659322978 2loss/SRqqt4/assignment_loss‚1.4081297300408704 1loss/FBt1/assignment_loss‚0.35121848441586473 /loss/FRt3/detection_loss‚1.9611105037885799 .loss/FRt4/detection_loss‚2.045726119840549 /loss/FBt1/detection_loss‚2.3964554156575884±—Õ—X“ÿ ‚(:ˆ ɵæÔŒ¯íü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.723' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.9446 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0n·Àe€!‚(:Ú ɵæÔÿ†ÿ disk.dm-1.out‚ 255.6015625 cpu‚25.32151790858486" disk.nvme0n1.out‚ 26728.1015625 disk.drbd1716.in‚0 disk.sda3.out‚ 255.6015625 disk.sda1.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd1716.out‚0) proc.memory.availableMB‚ 1314.97265625$ disk.nvme0c0n1.out‚ 26728.1015625 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 disk.rbd1.out‚36.03125 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0* proc.memory.percent‚15.674853324890137 network.sent‚ 2.0106342e+07* disk./.usagePercent‚20.312564769906164 disk.dm-0.in‚0.078125 disk.rbd1.in‚0$ memory_percent‚91.97404384613037 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.drbd2265.out‚0 network.recv‚ 2.368911e+06 disk.drbd2266.out‚0 disk.dm-0.out‚ 26728.1015625 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0# proc.memory.rssMB‚ 2568.16796875$ disk./.usageGB‚302.6718940734863 disk.sda.out‚ 255.6015625æÿbJ“!‚(:ˆ صæÔ Æ™ý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.342' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0684 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÎV/k‚!‚(:à صæÔ¾ñ¯ÿ disk.nvme0n1.in‚0.078125 disk.rbd0.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 1314.9765625 cpu‚25.320906523202268 proc.cpu.threads‚17 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd2080.out‚0% disk.nvme0c0n1.out‚26762.46484375 disk.dm-0.out‚26762.46484375$ disk./.usageGB‚302.6570053100586 disk.rbd1.in‚0 disk.drbd2080.in‚0$ memory_percent‚91.97402000427246* proc.memory.percent‚15.674853324890137 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 network.sent‚ 2.0214496e+07 disk.dm-1.in‚ 0.01171875 disk.sda.out‚ 256.21484375 disk.rbd1.out‚36.09375 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875# proc.memory.rssMB‚ 2568.16796875 network.recv‚ 2.375011e+06 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875* disk./.usagePercent‚20.311565572500022# disk.nvme0n1.out‚26762.46484375 disk.dm-1.out‚ 256.21484375 disk.sda3.out‚ 256.21484375 disk.sda2.in‚0 disk.sda2.out‚0ìŠc2ƒ!- _wandbruntime‚6668  _runtime‚6668{âÇYÇ„!Á _step‚690 & lr-AdamW/pg1‚3.189341787948786e-05 & lr-AdamW/pg2‚3.189341787948786e-05 trainer/global_step‚8449 % _timestamp‚1.7884515529388819e+09 _runtime‚6668.410478394\ü¸‰Ì…!Æ & lr-AdamW/pg2‚3.189341787948786e-05 trainer/global_step‚8449 % _timestamp‚1.7884515529388819e+09 _runtime‚6668.410478394 _step‚690 & lr-AdamW/pg1‚3.189341787948786e-05²p!¼2†!- _wandbruntime‚6668  _runtime‚6668>ù 4ç'‡!áO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 159268980 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚40291658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 170786816 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚490121823634944 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚542698738642944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚542698567856128 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚484335094975672 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚71483812 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 159289018 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚40292099 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 111775470 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 122527464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 281816484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚490121841637888 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 281796444 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚71483814 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7064470088704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚497186968089088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7412175959552 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 122527466 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚477296644746240 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7884515529395704e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚490121841637888 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚490121823634944 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 159289018 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 122527466 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7065126451200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7064470088704 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚497186293723648 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 122527464 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 111775913 trainer/global_step‚8449 _runtime‚6668.411385276 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚497186293723648 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 159268980 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚497186968089088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 146846720 _step‚691 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7038467268792 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚441 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚484335766452788 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚535286415836672 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚477296627706880 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7039121706548 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7065126451200 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7412322806272 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0åñ™oMˆ!æO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 159268980 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚40291658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 170786816 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚490121823634944 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚542698738642944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚542698567856128 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚484335094975672 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚71483812 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 159289018 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚40292099 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 111775470 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 122527464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 281816484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚490121841637888 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 281796444 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚71483814 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7064470088704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚497186968089088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7412175959552 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 122527466 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚477296644746240 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7884515529395704e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚490121841637888 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚490121823634944 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 159289018 Qjœ±ŸCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 122527466 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7065126451200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7064470088704 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚497186293723648 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 122527464 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 111775913 trainer/global_step‚8449 _runtime‚6668.411385276 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚443 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚497186293723648 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 159268980 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚497186968089088 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 146846720 _step‚691 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7038467268792 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚441 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚484335766452788 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚535286415836672 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚477296627706880 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7039121706548 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7065126451200 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7412322806272 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0³E³Ì2‰!- _wandbruntime‚6669  _runtime‚6669jO7ñ&Š!ëM OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 159307498 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚442 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7884515536381419e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚497244140569088 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7065945892352 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 122541480 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚490178194676736 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 122541482 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚497244140569088 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 281848980 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚40296175 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚490178176673792 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 281848980 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 _step‚692 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7039283703756 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 281828914 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7065945892352 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚490178176673792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 122541482 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7413201084416 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚535348042634240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚484390815541196 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 111788155 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 111788599 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚542761088496640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7065289516544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 159287434 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚490178194676736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚71491982 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚542761267658752 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 122541480 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚477351531837440 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7039938141720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7884515536381419e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚497244140569088 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7065945892352 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 122541480 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚490178194676736 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 122541482 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚497244140569088 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 281848980 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚40296175 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚490178176673792 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 281848980 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 _step‚692 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7039283703756 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 281828914 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7065945892352 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚490178176673792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 122541482 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7413201084416 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚535348042634240 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚484390815541196 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 111788155 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 111788599 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚542761088496640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7065289516544 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 159287434 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚490178194676736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚71491982 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚542761267658752 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 122541480 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚477351531837440 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7039938141720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 EDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 7068385754696 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚159 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 159904707 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚40583805 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 283141912 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 7094012577792 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚492715437590016 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚486863082543688 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚486862574589936 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚ 112500665 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 283157891 RDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚ 159888730 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚ 123253182 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚157 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚71916860 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 139283968 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.freed‚0 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚545447906371584 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 115343872 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚479794696788992 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚499809450167808 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844Rê¼ÙS*™!ÍT LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 159888730 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚545447767087616 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510833152 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚ 123253184 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 OADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚ 283141912 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914392 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚492715437590016 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15979 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15977 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 7442469265920 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 _runtime‚6727.766181951 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚ 283157891 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚492715455592960 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492830208 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9955586048 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚479794679749632 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955586048 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 WIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚ 112500824 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚40583962 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 337538560 jVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.allocated‚538005437105664 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚ 123253184 VHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚ 159904707 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15979 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 7067894840304 QIDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.allocated‚414 LDDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.peak‚908 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 trainer/global_step‚8462 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚499809961000960 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 7094505408000 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 _step‚694 \JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚ 7094012577792 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚538005413165568 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 RDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚ 123253182 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚492715437590016 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚486863082543688 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚486862574589936 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚ 112500665 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 283157891 RDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚ 159888730 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚ 123253182 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚157 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚71916860 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 139283968 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.freed‚0 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚545447906371584 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 115343872 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚479794696788992 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚499809450167808 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 TEDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.peak‚ 9643699025 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7884516122941391e+09 QGDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak‚24128 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 [GDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated‚499809961000960 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚499809450167808 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 7068385754696 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚159 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 159904707 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚40583805 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 283141912 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 7094012577792 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 trainer/global_step‚8462 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 hTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚538043312839168 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚71923366 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.freed‚40588529 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 YJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 \HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated‚499847156156416 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 LFDeviceStatsMonitor.on_validation_batch_end/oversize_segments.allocated‚0 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.current‚ 510833152 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 7094377704448 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚157 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚486898739912584 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚492752267618816 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚492752267618816 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 trainer/global_step‚8462 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 hTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚538043312839168 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚71923366 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.freed‚40588529 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 YJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 \HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated‚499847156156416 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 LFDeviceStatsMonitor.on_validation_batch_end/oversize_segments.allocated‚0 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.current‚ 510833152 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 7094377704448 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚157 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚486898739912584 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚492752267618816 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚492752267618816 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 B&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.5588235259056091 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.4047675132751465 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9877960085868835 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.019542373716831207 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9829459190368652 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 -"Purity/0FRt0SRqqt3FBt/event_purity‚0.0625 8"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.571871280670166 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.998068630695343 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 + Purity/1FRt0SRqqt3FBt/FRt_purity‚0.5625 *jet/accuracy_1_of_4‚0.9194027781486511 jet/accuracy_4_of_6‚0 particle/accuracy_5_of_6‚1 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.6458333134651184 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.19102942943572998 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9645417928695679 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5063300132751465 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5602075457572937 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 ) Purity/4FRt1SRqqt0FBt/FRt_purity‚0.25 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5015363097190857 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.15952380001544952 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.5904848575592041 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 +"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.25 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.2380952537059784 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 jet/accuracy_2_of_7‚0.5 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5484432578086853 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.12222222238779068 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.2939146161079407 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.3953407406806946 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.20267382264137268 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5789473652839661 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.14152158796787262 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 7 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.2505272924900055 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5751728415489197 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.17286935448646545 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.0943107008934021 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 8"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.520838737487793 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ) Purity/4FRt1SRqqt*FBt/FRt_purity‚0.25 _runtime‚6741.662102659 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5593173503875732 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.18981482088565826 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.2638888955116272 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.21875 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.6458333134651184 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.45228299498558044 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.3725694417953491 :"Purity/2FRt0SRqqt2FBt/event_purity‚0.00657894741743803 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 +jet/accuracy_2_of_5‚0.44290831685066223 particle/accuracy_3_of_6‚1 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 jet/accuracy_3_of_7‚0 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 jet/accuracy_5_of_5‚0 particle/accuracy_5_of_5‚1 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5789473652839661 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5054617524147034 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0912698432803154 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.49947136640548706 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9944282174110413 particle/accuracy_4_of_4‚1 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.42592594027519226 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2580982446670532 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3989933431148529 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 *jet/accuracy_1_of_1‚0.5114516615867615 +jet/accuracy_3_of_4‚0.04652384668588638 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.5866667032241821 +"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.25 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15475773811340332 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9744047522544861 8"Purity/2FRt*SRqqt*FBt/event_purity‚0.008843376301229 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.3333333432674408 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.10043187439441681 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.03497905284166336 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 + Purity/3FRt3SRqqt0FBt/FRt_purity‚0.1875 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 6 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.140216663479805 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1270817667245865 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 $ _timestamp‚1.788451626188302e+09 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.683577299118042 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5446698069572449 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.07356608659029007 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4305556118488312 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6098699569702148 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.05263157933950424 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5198444724082947 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 trainer/global_step‚8462 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3814113438129425 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.3431803584098816 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.11237373948097229 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 *jet/accuracy_2_of_4‚0.3682073652744293 8"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.543453574180603 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 :"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.46551722288131714 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.37485119700431824 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9365078806877136 8"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.455728679895401 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5372647047042847 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.03634961321949959 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 +particle/specificity‚0.9499956965446472 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.4583333730697632 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6091954708099365 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5816845297813416 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9852994680404663 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5749179124832153 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 jet/accuracy_1_of_6‚1 jet/accuracy_1_of_7‚1 9"Purity/*FRt0SRqqt0FBt/event_purity‚0.1476471871137619 7 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.1938459277153015 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 particle/accuracy_1_of_7‚1 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9785643219947815 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9375000596046448 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5798714756965637 jet/accuracy_5_of_7‚0 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 & Purity/*FRt0SRqqt4FBt/FRt_purity‚1 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5679469704627991 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5899810194969177 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.42447441816329956 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 +"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.25 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.012717309407889843 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9783507585525513 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 7 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.9899371266365051 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.4177631437778473 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9633619785308838 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.3355654776096344 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.13858941197395325 particle/accuracy_2_of_7‚1 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.24590083956718445 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 :"Purity/2FRt*SRqqt0FBt/event_purity‚0.00212565204128623 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.18081964552402496 7"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.53885418176651 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.21774737536907196 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9765334129333496 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.3125 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.2142857164144516 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 + Purity/3FRt3SRqqt*FBt/FRt_purity‚0.1875 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9896711111068726 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11314480006694794 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.4282945692539215 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.09654194116592407 particle/accuracy_1_of_3‚1 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9930239319801331 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5584365129470825 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9196217060089111 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.45164284110069275 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08423607051372528 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.23934492468833923 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4857759177684784 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 particle/accuracy_2_of_3‚1 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.030813809484243393 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.14639639854431152 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚1 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 8"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.560986340045929 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 ;"Purity/1FRt2SRqqt*FBt/event_purity‚0.000933706818614155 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.6969696879386902 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.4444444179534912 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.25840356945991516 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.015989631414413452 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 * Purity/4FRt*SRqqt0FBt/FRt_purity‚0.125 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 particle/accuracy_4_of_6‚1 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.2656852900981903 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9896175265312195 particle/accuracy_6_of_7‚1 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.4304713308811188 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 _step‚696 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9936092495918274 <"Purity/2FRt*SRqqt2FBt/event_purity‚0.0058139534667134285 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 *"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15156029164791107 ;"Purity/0FRt2SRqqt1FBt/event_purity‚0.028930816799402237 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.14299137890338898 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5489932298660278 +jet/accuracy_2_of_2‚0.10607486963272095 +jet/accuracy_2_of_3‚0.25065508484840393 particle/accuracy_2_of_2‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9923471212387085 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14069080352783203 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 jet/accuracy_6_of_6‚0 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9333333373069763 ("Purity/>«àÙjE0FRt4SRqqt0FBt/event_purity‚0 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 particle/accuracy_3_of_5‚1 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 jet/accuracy_7_of_7‚0 particle/accuracy_5_of_7‚1 )loss/val_total_loss‚45.47854671134638 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.26173338294029236 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9881795644760132 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9754397869110107 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5584365129470825 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5243934988975525 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5888389348983765 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.41036155819892883 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5805491805076599 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9867123961448669 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 *jet/accuracy_3_of_5‚0.0785074234008789 particle/accuracy_1_of_5‚1 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5325233340263367 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 6 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.903508722782135 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2916666865348816 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 particle/accuracy_3_of_7‚1 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9909546971321106 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.11522633582353592 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.3333333432674408 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.11484826356172562 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4270833432674408 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.31574001908302307 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 *jet/accuracy_1_of_5‚0.9639400839805603 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17784373462200165 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 ,"Purity/1FRt0SRqqt3FBt/event_purity‚0.125 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 *jet/accuracy_2_of_6‚0.3589743971824646 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.25485673546791077 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.02429625391960144 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 8"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.476688414812088 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.04014357179403305 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5702381134033203 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.13409961760044098 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_4‚1 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.14057281613349915 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.008881330490112305 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5525162816047668 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.5786375999450684 particle/accuracy_6_of_6‚1 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_1_of_6‚1 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4000000059604645 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 particle/accuracy_3_of_3‚1 particle/accuracy_2_of_6‚1 :"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.45816898345947266 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ,jet/accuracy_3_of_3‚0.020689763128757477 ,jet/accuracy_4_of_5‚0.004096262622624636 particle/accuracy_2_of_5‚1 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 * Purity/4FRt*SRqqt*FBt/FRt_purity‚0.125 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5627469420433044 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 ,jet/accuracy_4_of_4‚0.003799667814746499 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3474189043045044 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1758289933204651 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 (particle/accuracy‚0.8520017862319946 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9859274625778198 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.987687885761261 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.2708333432674408 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.21195803582668304 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9789801836013794 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.07092198729515076 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1885780245065689 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9628472924232483 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 +jet/accuracy_3_of_6‚0.17307692766189575 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5660443305969238 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.017025090754032135 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.15909090638160706 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5374890565872192 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5582351684570312 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3476601243019104 7 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.1793103665113449 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 +"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.25 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.14877848327159882 particle/accuracy_4_of_5‚1 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0972222313284874 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.3333333432674408 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1758289933204651 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 'particle/f_score‚0.4046163856983185 jet/accuracy_4_of_7‚0 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 8"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.636904776096344 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 particle/accuracy_4_of_7‚1 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6405608057975769 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 jet/accuracy_5_of_6‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.11286754161119461 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 8"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.575881838798523 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 8"Purity/0FRt0SRqqt*FBt/event_purity‚0.816524863243103 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.6666666269302368 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 8"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.548336386680603 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.19096991419792175 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5472221970558167 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.19274607300758362 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 7validation_average_jet_accuracy‚0.44948527216911316 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.3333333432674408 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.0984649509191513 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.19489845633506775 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9961155652999878 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5701976418495178 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.1632762998342514 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0014430014416575432 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5671296715736389 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5535714030265808 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.2499808818101883 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.08053731173276901 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5833333134651184 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.1384911984205246 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 6 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.345370352268219 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 *jet/accuracy_1_of_2‚0.7697300910949707 particle/accuracy_1_of_1‚1 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.4487539231777191 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5160744786262512 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.5990005731582642 7 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.2638889253139496 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9852607846260071 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.25632181763648987 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.1954365223646164 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 +particle/sensitivity‚0.3224223256111145 *jet/accuracy_1_of_3‚0.8860585689544678 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.17368420958518982 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.20253625512123108 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0972222313284874 jet/accuracy_6_of_7‚0 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4946392774581909 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9929894208908081 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 epoch‚6 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.47200435400009155 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4566666781902313 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.07410713285207748 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_7_of_7‚1 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.014367816969752312 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5204501152038574 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.6141974925994873 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5579558610916138 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3465828001499176 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.30669981241226196 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 particle/accuracy_1_of_4‚1 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 * Purity/0FRt2SRqqt2FBt/FBt_purity‚0.875 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5371876955032349 :"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.46481481194496155ÛgÞªˆ:¡!‚¦ ("Purity/4FRt2SRqqt*FBt/event_purity‚0 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.31574001908302307 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 *jet/accuracy_1_of_5‚0.9639400839805603 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17784373462200165 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 ,"Purity/1FRt0SRqqt3FBt/event_purity‚0.125 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 *jet/accuracy_2_of_6‚0.3589743971824646 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.25485673546791077 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.02429625391960144 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 8"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.476688414812088 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.04014357179403305 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5702381134033203 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.13409961760044098 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_4‚1 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.14057281613349915 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.008881330490112305 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5525162816047668 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.5786375999450684 particle/accuracy_6_of_6‚1 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_1_of_6‚1 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4000000059604645 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 particle/accuracy_3_of_3‚1 particle/accuracy_2_of_6‚1 :"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.45816898345947266 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ,jet/accuracy_3_of_3‚0.020689763128757477 ,jet/accuracy_4_of_5‚0.004096262622624636 particle/accuracy_2_of_5‚1 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 * Purity/4FRt*SRqqt*FBt/FRt_purity‚0.125 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5627469420433044 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 ,jet/accuracy_4_of_4‚0.003799667814746499 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3474189043045044 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1758289933204651 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 (particle/accuracy‚0.8520017862319946 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9859274625778198 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.987687885761261 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.2708333432674408 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.21195803582668304 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9789801836013794 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.07092198729515076 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1885780245065689 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9628472924232483 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 +jet/accuracy_3_of_6‚0.17307692766189575 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5660443305969238 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.017025090754032135 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.15909090638160706 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5374890565872192 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5582351684570312 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3476601243019104 7 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.1793103665113449 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 +"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.25 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.14877848327159882 particle/accuracy_4_of_5‚1 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0972222313284874 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.3333333432674408 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1758289933204651 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 'particle/f_score‚0.4046163856983185 jet/accuracy_4_of_7‚0 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 8"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.636904776096344 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 particle/accuracy_4_of_7‚1 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6405608057975769 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 jet/accuracy_5_of_6‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.11286754161119461 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 8"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.575881838798523 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 8"Purity/0FRt0SRqqt*FBt/event_purity‚0.816524863243103 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.6666666269302368 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 8"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.548336386680603 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.19096991419792175 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5472221970558167 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.19274607300758362 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 7validation_average_jet_accuracy‚0.44948527216911316 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 9"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.3333333432674408 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.0984649509191513 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.19489845633506775 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9961155652999878 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5701976418495178 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.1632762998342514 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0014430014416575432 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5671296715736389 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5535714030265808 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.2499808818101883 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.08053731173276901 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5833333134651184 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.1384911984205246 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 6 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.345370352268219 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 *jet/accuracy_1_of_2‚0.7697300910949707 particle/accuracy_1_of_1‚1 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.4487539231777191 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5160744786262512 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.5990005731582642 7 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.2638889253139496 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9852607846260071 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.25632181763648987 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.1954365223646164 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 +particle/sensitivity‚0.3224223256111145 *jet/accuracy_1_of_3‚0.8860585689544678 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.17368420958518982 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.20253625512123108 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0972222313284874 jet/accuracy_6_of_7‚0 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4946392774581909 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9929894208908081 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 epoch‚6 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.47200435400009155 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4566666781902313 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.07410713285207748 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_7_of_7‚1 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.014367816969752312 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5204501152038574 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.6141974925994873 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5579558610916138 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3465828001499176 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.30669rnÖôX981241226196 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 particle/accuracy_1_of_4‚1 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 * Purity/0FRt2SRqqt2FBt/FBt_purity‚0.875 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5371876955032349 :"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.46481481194496155 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9672514796257019 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.1111111119389534 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.5588235259056091 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.4047675132751465 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9877960085868835 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.019542373716831207 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9829459190368652 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 -"Purity/0FRt0SRqqt3FBt/event_purity‚0.0625 8"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.571871280670166 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.998068630695343 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 + Purity/1FRt0SRqqt3FBt/FRt_purity‚0.5625 *jet/accuracy_1_of_4‚0.9194027781486511 jet/accuracy_4_of_6‚0 particle/accuracy_5_of_6‚1 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.6458333134651184 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.19102942943572998 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9645417928695679 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5063300132751465 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5602075457572937 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 ) Purity/4FRt1SRqqt0FBt/FRt_purity‚0.25 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5015363097190857 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.15952380001544952 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.5904848575592041 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 +"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.25 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.2380952537059784 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 jet/accuracy_2_of_7‚0.5 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5484432578086853 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.12222222238779068 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.2939146161079407 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.3953407406806946 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.20267382264137268 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5789473652839661 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.14152158796787262 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 7 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.2505272924900055 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5751728415489197 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.17286935448646545 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.0943107008934021 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 8"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.520838737487793 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ) Purity/4FRt1SRqqt*FBt/FRt_purity‚0.25 _runtime‚6741.662102659 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5593173503875732 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.18981482088565826 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.2638888955116272 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.21875 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.6458333134651184 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.45228299498558044 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.3725694417953491 :"Purity/2FRt0SRqqt2FBt/event_purity‚0.00657894741743803 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 +jet/accuracy_2_of_5‚0.44290831685066223 particle/accuracy_3_of_6‚1 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 jet/accuracy_3_of_7‚0 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 jet/accuracy_5_of_5‚0 particle/accuracy_5_of_5‚1 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.5789473652839661 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5054617524147034 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0912698432803154 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.49947136640548706 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9944282174110413 particle/accuracy_4_of_4‚1 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.42592594027519226 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2580982446670532 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.3989933431148529 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 *jet/accuracy_1_of_1‚0.5114516615867615 +jet/accuracy_3_of_4‚0.04652384668588638 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.5866667032241821 +"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.25 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15475773811340332 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9744047522544861 8"Purity/2FRt*SRqqt*FBt/event_purity‚0.008843376301229 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 9"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.3333333432674408 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.10043187439441681 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.03497905284166336 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 + Purity/3FRt3SRqqt0FBt/FRt_purity‚0.1875 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 6 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.140216663479805 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1270817667245865 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 $ _timestamp‚1.788451626188302e+09 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.683577299118042 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5446698069572449 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.07356608659029007 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4305556118488312 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6098699569702148 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.05263157933950424 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5198444724082947 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 trainer/global_step‚8462 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3814113438129425 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.3431803584098816 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.11237373948097229 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 *jet/accuracy_2_of_4‚0.3682073652744293 8"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.543453574180603 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 :"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.46551722288131714 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.37485119700431824 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9365078806877136 8"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.455728679895401 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5372647047042847 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.03634961321949959 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 +particle/specificity‚0.9499956965446472 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.4583333730697632 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6091954708099365 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5816845297813416 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9852994680404663 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5749179124832153 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 jet/accuracy_1_of_6‚1 jet/accuracy_1_of_7‚1 9"Purity/*FRt0SRqqt0FBt/event_purity‚0.1476471871137619 7 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.1938459277153015 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 particle/accuracy_1_of_7‚1 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9785643219947815 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9375000596046448 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5798714756965637 jet/accuracy_5_of_7‚0 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 & Purity/*FRt0SRqqt4FBt/FRt_purity‚1 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5679469704627991 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5899810194969177 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.42447441816329956 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 +"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.25 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.012717309407889843 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9783507585525513 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 7 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.9899371266365051 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.4177631437778473 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9633619785308838 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.3355654776096344 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.13858941197395325 particle/accuracy_2_of_7‚1 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.24590083956718445 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 :"Purity/2FRt*SRqqt0FBt/event_purity‚0.00212565204128623 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.18081964552402496 7"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.53885418176651 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.21774737536907196 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9765334129333496 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.3125 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.2142857164144516 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 + Purity/3FRt3SRqqt*FBt/FRt_purity‚0.1875 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9896711111068726 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11314480006694794 7 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.4282945692539215 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.09654194116592407 particle/accuracy_1_of_3‚1 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9930239319801331 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5584365129470825 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9196217060089111 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.45164284110069275 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08423607051372528 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.23934492468833923 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4857759177684784 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 particle/accuracy_2_of_3‚1 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.030813809484243393 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.14639639854431152 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚1 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 8"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.560986340045929 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 ;"Purity/1FRt2SRqqt*FBt/event_purity‚0.000933706818614155 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.6969696879386902 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.4444444179534912 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.25840356945991516 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.015989631414413452 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 * Purity/4FRt*SRqqt0FBt/FRt_purity‚0.125 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 particle/accuracy_4_of_6‚1 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.2656852900981903 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9896175265312195 particle/accuracy_6_of_7‚1 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.4304713308811188 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 _step‚696 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9936092495918274 <"Purity/2FRt*SRqqt2FBt/event_purity‚0.0058139534667134285 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 *"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15156029164791107 ;"Purity/0FRt2SRqqt1FBt/event_purity‚0.028930816799402237 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.14299137890338898 ;"Purity/1FRt1SRqqt2FBt/event_purity‚0.010416666977107525 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5489932298660278 +jet/accuracy_2_of_2‚0.10607486963272095 +jet/accuracy_2_of_3‚0.25065508484840393 particle/accuracy_2_of_2‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9923471212387085 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14069080352783203 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 jet/accuracy_6_of_6‚0 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9333333373069763 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 particle/accuracy_3_of_5‚1 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 jet/accuracy_7_of_7‚0 particle/accuracy_5_of_7‚1 )loss/val_total_loss‚45.47854671134638 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.26173338294029236 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9881795644760132 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9754397869110107 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5584365129470825 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5243934988975525 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5888389348983765 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.41036155819892883 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5805491805076599 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9867123961448669 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 *jet/accuracy_3_of_5‚0.0785074234008789 particle/accuracy_1_of_5‚1 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5325233340263367 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 6 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.903508722782135 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2916666865348816 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 particle/accuracy_3_of_7‚1 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9909546971321106 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.11522633582353592 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.3333333432674408 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.11484826356172562 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4270833432674408¸‰W¾“¢!‚(:ˆ ²¶æÔÄ“šý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚128.801' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.7602 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0]Ô¯o£!‚(:ä ²¶æÔçÑ°ÿ$ disk.nvme0n1.out‚27488.783203125 disk.dm-1.out‚ 259.57421875* disk./.usagePercent‚20.314146128445167 disk.drbd2080.in‚0! disk.dm-0.out‚27488.783203125 disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.rbd1.out‚ 36.7890625 disk.drbd1716.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 proc.cpu.threads‚17& disk.nvme0c0n1.out‚27488.783203125 disk.sda3.out‚ 259.57421875 disk.rbd0.in‚0$ disk./.usageGB‚302.6954574584961 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875 network.sent‚ 2.0454713e+07 disk.sda.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0$ memory_percent‚90.62342643737793 cpu‚25.361747246742926 network.recv‚ 2.404232e+06 disk.sda.out‚ 259.57421875 disk.sda2.out‚0( proc.memory.availableMB‚ 1536.2578125* proc.memory.percent‚16.001558303833008 disk.drbd2266.out‚0BFõÅ“¤!‚(:ˆ Á¶æÔòÿŒý gpu.0.gpu‚27.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚179.482' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8964 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0õiYn¥!‚(:ã Á¶æÔ­ú¢€ disk.drbd2265.out‚0 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚27574.79150390625 disk.rbd0.out‚0# disk.dm-0.out‚27574.79150390625 disk.sda.in‚ 0.01171875& disk.nvme0n1.out‚27574.79150390625' proc.memory.availableMB‚ 1536.515625" proc.memory.rssMB‚ 2621.6953125 network.sent‚ 2.0577879e+07 disk.dm-1.out‚ 260.140625 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0$ memory_percent‚90.62185287475586 disk.sda3.out‚ 260.140625 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125% disk./.usageGB‚302.69922256469727 disk.dm-1.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.sda.out‚ 260.140625 disk.sda1.out‚0 disk.rbd1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd1.out‚ 36.7890625 cpu‚25.361015320790823) disk./.usagePercent‚20.31439880788146 disk.drbd1716.out‚0* proc.memory.percent‚16.001558303833008 proc.cpu.threads‚17 network.recv‚ 2.409987e+06 disk.drbd2265.in‚0Ã_Äq2¦!- _wandbruntime‚6772  _runtime‚6772• ɧ!à _step‚697 ' lr-AdamW/pg1‚3.1102964536967015e-05 ' lr-AdamW/pg2‚3.1102964536967015e-05 trainer/global_step‚8499 % _timestamp‚1.7884516572691827e+09 _runtime‚6772.740792512&¾oåΨ!È ' lr-AdamW/pg1‚3.1102964536967015e-05 ' lr-AdamW/pg2‚3.1102964536967015e-05 trainer/global_step‚8499 % _timestamp‚1.7884516572691827e+09 _runtime‚6772.740792512 _step‚697¹&óIÊ2©!- _wandbruntime‚6772  _runtime‚6772¸Ý¼Þ;ª!áO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 160677033 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _step‚698 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7478587203584 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚72301964 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚495259785394688 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7129132227072 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚72301966 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 123909044 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 284586077 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚548233525667328 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚540755099990528 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 123909044 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 160656995 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7128475864576 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚482265341082624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 284566037 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚495259785394688 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚495259767391744 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚40811065 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚445 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚482265358121984 _runtime‚6772.742112293 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚495259767391744 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚489368239668022 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7102881546038 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚502388243256320 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 113113031 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 123909042 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 113112584 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚40810620 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7478425676800 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚489367568190906 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 160677033 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430**Ùì'«!æO M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 160656995 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚502388243256320 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 113113031 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 123909042 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 113112584 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚40810620 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7478425676800 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚489367568190906 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 160677033 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 160677033 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _step‚698 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7478587203584 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚72301964 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚495259785394688 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7129132227072 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚72301966 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 123909044 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 284586077 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚548233525667328 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚540755099990528 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 123909044 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 160656995 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7128475864576 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚482265341082624 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 284566037 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚495259785394688 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚495259767391744 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚40811065 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚445 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚482265358121984 _runtime‚6772.742112293 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚495259767391744 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚489368239668022 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7102881546038 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7479448569344 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _step‚699 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7129295292416 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚495316138433536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 284618573 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7129295292416 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 123923060 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚495316120430592 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 113125658 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚448 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚40815524 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 123923060 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚540816774668288 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚502446090101760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚540816750728192 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 160695513 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚548296037784064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 284598507 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 123923058 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7129951668224 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚489423288756430 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚482320262252544 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚495316138433536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚495316120430592 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚482320245213184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 160675449 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7129951668224 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚40815078 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 284618573 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7103697981210 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚502445415723008 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 160695513 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 123923058 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 113125210 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚72310132 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7479287055872ÞB#"®!ïM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7129295292416 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚495316138433536 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 284618573 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7129295292416 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 123923060 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚495316120430592 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 113125658 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚448 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚40815524 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 123923060 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚540816774668288 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚502446090101760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚540816750728192 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 160695513 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚548296037784064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 284598507 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 123923058 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7129951668224 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚489423288756430 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚482320262252544 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚495316138433536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚495316120430592 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚482320245213184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 160675449 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7129951668224 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚40815078 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 284618573 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7103697981210 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚502445415723008 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 160695513 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 123923058 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 113125210 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚72310132 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7479287055872 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚502446090101760 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7479448569344 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _step‚699»M2¯!- _wandbruntime‚6773  _runtime‚6773Z¬‘Z°!Ô /loss/FRt3/detection_loss‚1.8708245634120337 1loss/SRqqt4/detection_loss‚1.7230472535479302 &loss/total_loss‚46.266092658494216 trainer/global_step‚8499 .loss/FRt1/detection_loss‚3.434558281302452 .loss/FRt2/detection_loss‚2.399222004981268 .loss/FBt2/detection_loss‚2.445719155669212 _runtime‚6773.447029786 2loss/SRqqt3/assignment_loss‚0.9904797004447097 1loss/FBt2/assignment_loss‚0.32125410735605653 /loss/FBt3/detection_loss‚0.6677143259710351 1loss/FBt4/assignment_loss‚0.45147912204265594 /loss/SRqqt1/detection_loss‚2.67521490432598 epoch‚7 1loss/FBt1/assignment_loss‚0.49003088158739655 1loss/SRqqt2/detection_loss‚1.8518513382102053 1loss/SRqqt3/detection_loss‚1.1841805469067324 .loss/FBt4/detection_loss‚2.880423684914907 _step‚700 0loss/FRt2/assignment_loss‚3.1444114858195897 1loss/FBt3/assignment_loss‚0.13781195585268385 % _timestamp‚1.7884516579755151e+09 /loss/FRt1/assignment_loss‚4.269913613796234 /loss/FRt4/detection_loss‚2.4322244748473167 0loss/FRt3/assignment_loss‚1.3875766068088764 /loss/FRt4/assignment_loss‚3.119795237075199 2loss/SRqqt1/assignment_loss‚2.1139812292876066 2loss/SRqqt2/assignment_loss‚1.3823533076016854 1loss/SRqqt4/assignment_loss‚1.382473341682378 .loss/FBt1/detection_loss‚3.509551535050074D%®Ô_±!Ù 1loss/FBt2/assignment_loss‚0.32125410735605653 /loss/FBt3/detection_loss‚0.6677143259710351 1loss/FBt4/assignment_loss‚0.45147912204265594 /loss/SRqqt1/detection_loss‚2.67521490432598 epoch‚7 1loss/FBt1/assignment_loss‚0.49003088158739655 1loss/SRqqt2/detection_loss‚1.8518513382102053 1loss/SRqqt3/detection_loss‚1.1841805469067324 .loss/FBt4/detection_loss‚2.880423684914907 _step‚700 0loss/FRt2/assignment_loss‚3.1444114858195897 1loss/FBt3/assignment_loss‚0.13781195585268385 % _timestamp‚1.7884516579755151e+09 /loss/FRt1/assignment_loss‚4.269913613796234 /loss/FRt4/detection_loss‚2.4322244748473167 0loss/FRt3/assignment_loss‚1.3875766068088764 /loss/FRt4/assignment_loss‚3.119795237075199 2loss/SRqqt1/assignment_loss‚2.1139812292876066 2loss/SRqqt2/assignment_loss‚1.3823533076016854 1loss/SRqqt4/assignment_loss‚1.382473341682378 .loss/FBt1/detection_loss‚3.509551535050074 /loss/FRt3/detection_loss‚1.8708245634120337 1loss/SRqqt4/detection_loss‚1.7230472535479302 &loss/total_loss‚46.266092658494216 trainer/global_step‚8499 .loss/FRt1/detection_loss‚3.434558281302452 .loss/FRt2/detection_loss‚2.399222004981268 .loss/FBt2/detection_loss‚2.445719155669212 _runtime‚6773.447029786 2loss/SRqqt3/assignment_loss‚0.9904797004447097¼Ò$“²!‚(:ˆ жæÔÀ´•ý gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚180.141' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0282 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0PD-q³!‚(:æ жæԤ³ÿ disk.drbd2080.out‚0' disk.nvme0c0n1.out‚28035.6455078125 network.recv‚ 2.414779e+06 disk.dm-0.in‚0.078125" disk.dm-0.out‚28035.7900390625 disk.rbd1.out‚ 36.8212890625 disk.sda3.in‚ 0.01171875* proc.memory.percent‚16.001558303833008$ disk./.usageGB‚302.7031593322754 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.sda1.out‚0 disk.sda2.out‚0* disk./.usagePercent‚20.314663007656797 disk.drbd1716.out‚0 disk.drbd2266.out‚0$ memory_percent‚90.62671661376953 disk.dm-1.out‚ 260.70703125 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 network.sent‚ 2.0584301e+07 disk.sda2.in‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚28035.7900390625 disk.sda3.out‚ 260.70703125 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 cpu‚25.3603001502059 disk.dm-1.in‚ 0.01171875& proc.memory.availableMB‚ 1535.71875 disk.sda.out‚ 260.70703125 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17[rv“´!‚(:ˆ ߶æÔÍã…ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.262' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.8524 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0|Žªgµ!‚(:Ü ߶æÔ¨Š÷ÿ network.sent‚ 2.0692006e+07 disk.rbd0.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125' proc.memory.availableMB‚ 1536.453125 proc.cpu.threads‚17 disk.sda.out‚261.1875 disk.sda1.out‚0% disk.nvme0n1.out‚28062.9345703125' disk.nvme0c0n1.out‚28062.9345703125" proc.memory.rssMB‚ 2621.6953125 disk.rbd0.in‚0 disk.dm-1.out‚261.1875 disk.drbd2265.in‚0 disk.sda3.out‚261.1875" disk.dm-0.out‚28062.9345703125 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 network.recv‚ 2.419854e+06 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2266.out‚0$ memory_percent‚90.62223434448242) disk./.usagePercent‚20.31454498818351% disk./.usageGB‚302.70140075683594 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.rbd1.out‚ 36.9462890625 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0* proc.memory.percent‚16.001558303833008 disk.drbd2080.out‚0 cpu‚25.35965071694283¬Ž»½2¶!- _wandbruntime‚6809  _runtime‚6809ˆú%èÇ·!Á & lr-AdamW/pg1‚3.031922993749945e-05 & lr-AdamW/pg2‚3.031922993749945e-05 trainer/global_step‚8549 % _timestamp‚1.7884516943585138e+09 _runtime‚6809.830153321 _step‚701!™˜e̸!Æ & lr-AdamW/pg1‚3.031922993749945e-05 & lr-AdamW/pg2‚3.031922993749945e-05 trainer/global_step‚8549 % _timestamp‚1.7884516943585138e+09 _runtime‚6809.830153321 _step‚701½I"62¹!- _wandbruntime‚6809  _runtime‚6809§h]rç'º!áO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7170118032896 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚485010564649984 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 124609842 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 161603085 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 161583047 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7521659758080 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7169461670400 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚485010547610624 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 286192889 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚8549 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚702 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 161583047 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7170118032896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚498077437337088 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 113746611 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚498077419334144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚505246881004544 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 113746169 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚492154281535430 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 124609842 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚440 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚551359320571904 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7143062447690 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚505247555369984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚505246881004544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928T…\ ì'»!æO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 124609842 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚440 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚551359320571904 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7143062447690 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚505247555369984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚505246881004544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7170118032896 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚485010564649984 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 124609842 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 161603085 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 161583047 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7521659758080 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7169461670400 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚485010547610624 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 286192889 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚8549 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚702 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 161583047 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7170118032896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚498077437337088 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 113746611 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚498077419334144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚505246881004544 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 113746169 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚492154281535430¾DxÿÁ“¼!‚(:ˆ î¶æÔ£³Ÿý gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚186.523' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.3046 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0A>w½!‚(:ì î¶æÔؘ°€* disk./.usagePercent‚20.315008873835556% disk./.usageGB‚302.70831298828125 disk.sda.in‚ 0.01171875$ memory_percent‚90.62678813934326 network.sent‚ 2.0697931e+07 disk.drbd2265.out‚0 disk.sda2.in‚0 proc.cpu.threads‚17% disk.nvme0n1.out‚28100.9560546875 disk.sda2.out‚0 cpu‚25.358968140204755 disk.sda1.in‚0 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚28100.9560546875 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875" disk.dm-0.out‚28100.9560546875 disk.sda.out‚ 261.80078125 disk.rbd1.out‚ 37.0087890625) proc.memory.availableMB‚ 1535.70703125* proc.memory.percent‚16.001558303833008 network.recv‚ 2.424719e+06 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.dm-1.out‚ 261.80078125 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.sda3.out‚ 261.80078125" proc.memory.rssMB‚ 2621.6953125=ý2¾!- _wandbruntime‚6810  _runtime‚6810œ»Á ¿!ëM M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚505304727849984 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 161601501 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7522520908800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚492210002101162 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7144533320618 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚8549 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 161621565 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚498133772372992 ]KDeviceStatsMonitor.on_train_batch_end/inact¸˜þ×0ive_split_bytes.small_pool.freed‚ 7522365686784 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 286245425 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚498133790375936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚6810.525380019 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 124623860 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 113758751 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 161601501 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚551421832460288 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚425 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚551421653298176 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 286225359 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚72718534 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚492209330623838 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7170937474048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚505304727849984 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚423 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚485065451741184 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚41040642 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 % _timestamp‚1.7884516950533714e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚498133790375936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚505304053471232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚498133772372992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401'ò>ö&À!ðM PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 286245425 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚498133790375936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚6810.525380019 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 124623860 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 113758751 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 161601501 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚551421832460288 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚425 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚551421653298176 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 286225359 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚72718534 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚492209330623838 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7170937474048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚505304727849984 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚423 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚485065451741184 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚41040642 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 % _timestamp‚1.7884516950533714e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚498133790375936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚505304053471232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚498133772372992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚505304727849984 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 161601501 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7522520908800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚492210002101162 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7144533320618 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚8549 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 161621565 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚498133772372992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7522365686784 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BûåÇÉ!Á & lr-AdamW/pg2‚2.954237741020101e-05 trainer/global_step‚8599 % _timestamp‚1.7884517316921868e+09 _runtime‚6847.163773556 _step‚705 & lr-AdamW/pg1‚2.954237741020101e-05†è‹þÌÊ!Æ _step‚705 & lr-AdamW/pg1‚2.954237741020101e-05 & lr-AdamW/pg2‚2.954237741020101e-05 trainer/global_step‚8599 % _timestamp‚1.7884517316921868e+09 _runtime‚6847.163773556ÁÄÿ62Ë!- _wandbruntime‚6847  _runtime‚6847î‰e{$Ì!àO ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 162509099 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚554484712635392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7564342925824 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚500895071276544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚546920197697024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚487755754138624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚500895071276544 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚449 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 125310644 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7210447476224 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚73118766 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 162529137 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41260634 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 125310642 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7211103838720 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚494940323402838 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚487755771177984 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚494939651925722 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7184552224854 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 125310642 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7210447476224 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚554484516682752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 % _timestamp‚1.7884517316929064e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7183897787098 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚508106193118208 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚73118764 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚500895089279488 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚508105518752768 UADeviceStatsMonitor.on_train_batch_start/aluu&klocated_bytes.all.freed‚508105518752768 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7564514938368 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚8599 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚546920173756928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0~W'Žë'Í!åO WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7210447476224 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚73118766 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 162529137 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41260634 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 125310642 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7211103838720 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚494940323402838 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚487755771177984 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚494939651925722 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7184552224854 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 125310642 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7210447476224 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚554484516682752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 % _timestamp‚1.7884517316929064e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7183897787098 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚508106193118208 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚73118764 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚500895089279488 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚508105518752768 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚508105518752768 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7564514938368 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚8599 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚546920173756928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 162509099 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚554484712635392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7564342925824 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚500895071276544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚546920197697024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚487755754138624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚500895071276544 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚449 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 125310644 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952Âœ]Té2Î!- _wandbruntime‚6847  _runtime‚6847»àðñ&Ï!ëM K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 125324658 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 162547617 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 162527553 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7211266904064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚41265528 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚500951424315392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚500951424315392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 114392013 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7565361833984 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚487810658269184 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 287852211 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 125324660 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚73126934 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚494996043968570 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 287872277 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7211923279872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7211266904064 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 162527553 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7185368660026 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7184714222062 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚546981848434688 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚554547210268672 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚508163365598208 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚554547012232192 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 114392462 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚508162691219456 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884517324006448e+09 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚508163365598208 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 _step‚707 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 162547617 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚6847.872702134 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 287872277 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚494995372491246 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚41265081 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944ÜqÞö&Ð!ðM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚73126934 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚494996043968570 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 287872277 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7211923279872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7211266904064 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 162527553 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7185368660026 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7184714222062 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚546981848434688 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚554547210268672 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚508163365598208 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚554547012232192 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 114392462 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚508162691219456 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884517324006448e+09 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚508163365598208 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 _step‚707 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 162547617 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚6847.872702134 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 287872277 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚494995372491246 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚41265081 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 125324658 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 162547617 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 162527553 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7211266904064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚41265528 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚500951424315392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚500951424315392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 114392013 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7565361833984 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚487810658269184 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 287852211 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 125324660 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0Ã-fÊ2Ñ!- _wandbruntime‚6847  _runtime‚6847E¤UÒ!Ï .loss/FRt4/detection_loss‚2.175154213865216 /loss/FBt1/detection_loss‚3.0570185780525208 .loss/FBt4/detection_loss‚2.607042213280996 trainer/global_step‚8599 /loss/FRt4/assignment_loss‚3.202243467936149 1loss/SRqqt4/detection_loss‚1.8679306380292202 .loss/FBt2/detection_loss‚2.338616377540997 0loss/SRqqt1/assignment_loss‚1.69562541770524 /loss/FRt1/detection_loss‚3.1095234277071775 epoch‚7 _step‚708 2loss/SRqqt3/assignment_loss‚1.1639954268873212 2loss/SRqqt4/assignment_loss‚1.5423433213157858 /loss/FBt3/assignment_loss‚0.259591484868137 1loss/SRqqt2/detection_loss‚1.9365194287411003 2loss/SRqqt2/assignment_loss‚1.1431886915030867 0loss/SRqqt1/detection_loss‚2.553856471489216 $ _timestamp‚1.788451732402383e+09 _runtime‚6847.873953757 0loss/FRt1/assignment_loss‚3.9923941647564924 1loss/SRqqt3/detection_loss‚1.0620492670744184 0loss/FBt1/assignment_loss‚0.6063626158550051 1loss/FBt2/assignment_loss‚0.45156184530682303 0loss/FBt4/assignment_loss‚0.5405324005211393 .loss/FBt3/detection_loss‚0.572554602341234 %loss/total_loss‚44.16961112118904 0loss/FRt2/assignment_loss‚2.9952578580740727 0loss/FRt3/assignment_loss‚1.1478673652106641 .loss/FRt2/detection_loss‚2.488842792357459 /loss/FRt3/detection_loss‚1.6595390507695722ÜoˆVÒÓ!Ô 1loss/SRqqt3/detection_loss‚1.0620492670744184 0loss/FBt1/assignment_loss‚0.6063626158550051 1loss/FBt2/assignment_loss‚0.45156184530682303 0loss/FBt4/assignment_loss‚0.5405324005211393 .ùβ§ˆloss/FBt3/detection_loss‚0.572554602341234 %loss/total_loss‚44.16961112118904 0loss/FRt2/assignment_loss‚2.9952578580740727 0loss/FRt3/assignment_loss‚1.1478673652106641 .loss/FRt2/detection_loss‚2.488842792357459 /loss/FRt3/detection_loss‚1.6595390507695722 .loss/FRt4/detection_loss‚2.175154213865216 /loss/FBt1/detection_loss‚3.0570185780525208 .loss/FBt4/detection_loss‚2.607042213280996 trainer/global_step‚8599 /loss/FRt4/assignment_loss‚3.202243467936149 1loss/SRqqt4/detection_loss‚1.8679306380292202 .loss/FBt2/detection_loss‚2.338616377540997 0loss/SRqqt1/assignment_loss‚1.69562541770524 /loss/FRt1/detection_loss‚3.1095234277071775 epoch‚7 _step‚708 2loss/SRqqt3/assignment_loss‚1.1639954268873212 2loss/SRqqt4/assignment_loss‚1.5423433213157858 /loss/FBt3/assignment_loss‚0.259591484868137 1loss/SRqqt2/detection_loss‚1.9365194287411003 2loss/SRqqt2/assignment_loss‚1.1431886915030867 0loss/SRqqt1/detection_loss‚2.553856471489216 $ _timestamp‚1.788451732402383e+09 _runtime‚6847.873953757 0loss/FRt1/assignment_loss‚3.9923941647564924Ä‚8Be“Ô!‚(:ˆ ›·æÔ‚È©ý gpu.0.gpu‚27.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.951' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.7902 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡SSuÕ!‚(:ê ›·æÔ”ÝÎÿ disk.drbd2266.in‚0' proc.memory.availableMB‚ 1534.359375 proc.cpu.threads‚17 disk.sda1.out‚0 disk.rbd1.out‚ 37.166015625 disk.nvme0c0n1.in‚0.078125 network.recv‚ 2.44036e+06$ disk./.usageGB‚302.7238311767578 disk.sda.out‚ 263.13671875& disk.nvme0n1.out‚28211.15673828125 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0* proc.memory.percent‚16.001558303833008* disk./.usagePercent‚20.316050312484847 disk.dm-0.in‚0.078125# disk.dm-0.out‚28211.15673828125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0$ memory_percent‚90.63501358032227 network.sent‚ 2.0897994e+07 disk.sda.in‚ 0.01171875 disk.drbd1716.out‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚28211.15673828125 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 263.13671875" proc.memory.rssMB‚ 2621.6953125 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 263.13671875 disk.rbd0.in‚0 disk.drbd2265.out‚0 disk.rbd0.out‚0 cpu‚25.356917949699696=Ù’Ö!‚(:‡ ª·æÔÕ„øü gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚184.654' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9308 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ä\¸îo×!‚(:ä ª·æÔó›ýþ network.recv‚ 2.44827e+06 disk.drbd2266.in‚0 disk.sda.out‚ 263.76953125 disk.sda1.out‚0 disk.rbd1.out‚ 37.228515625 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125) disk./.usagePercent‚20.31756126895193 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 263.76953125% disk./.usageGB‚302.74634552001953 disk.drbd2266.out‚0 disk.dm-1.out‚ 263.76953125 disk.drbd2265.out‚0' proc.memory.availableMB‚ 1534.296875* proc.memory.percent‚16.001558303833008 network.sent‚ 2.1007366e+07 disk.sda1.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0& disk.nvme0c0n1.out‚28249.357421875 cpu‚25.356255627502687 proc.cpu.threads‚17 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125! disk.dm-0.out‚28249.357421875 disk.nvme0n1.in‚0.078125$ disk.nvme0n1.out‚28249.357421875$ memory_percent‚90.63539505004883¸1ÇI2Ø!- _wandbruntime‚6884  _runtime‚6884˜ÙùýÉÙ!à _runtime‚6884.684700296 _step‚709 ' lr-AdamW/pg1‚2.8772568849974014e-05 ' lr-AdamW/pg2‚2.8772568849974014e-05 trainer/global_step‚8649 % _timestamp‚1.7884517692131023e+09ž×ÉMÎÚ!È _runtime‚6884.684700296 _step‚709 ' lr-AdamW/pg1‚2.8772568849974014e-05 ' lr-AdamW/pg2‚2.8772568849974014e-05 trainer/global_step‚8649 % _timestamp‚1.7884517692131023e+09ÅÌF^2Û!- _wandbruntime‚6884  _runtime‚6884ÕjÆdç'Ü!áO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚550002710640128 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 163435151 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 163455189 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 _step‚710 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚510964156500992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7607240628224 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884517692138028e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚41487684 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚510964156500992 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 126011444 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 174109696 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 126011442 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚557610149318144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7225387564262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 115014850 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚550002734580224 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚503712723218944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚73527166 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 126011444 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7251433282048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 163455189 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7252089644544 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7607414737920 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚497725693793130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 289446593 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7251433282048 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚503712723218944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41487230 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚557609951268352 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚490500977705984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚510964830866432 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 163435151 trainer/global_step‚8649Êí“ì'Ý!æO QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 _step‚710 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚510964156500992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7607240628224 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 % _timestamp‚1.7884517692138028e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚41487684 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚510964156500992 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 126011444 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 174109696 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 126011442 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚557610149318144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7225387564262 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 115014850 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚550002734580224 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚503712723218944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚73527166 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 126011444 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7251433282048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 163455189 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7252089644544 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7607414737920 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚497725693793130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 289446593 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7251433282048 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚503712723218944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41487230 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚557609951268352 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚490500977705984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚510964830866432 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 163435151 trainer/global_step‚8649 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚550002710640128 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 163435151 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 163455189 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0Æ„JšîžÞ!‚(:“ ¹·æÔŸØ—ý gpu.0.gpu‚27.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚170.842' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚34.168400000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‹/HÊkß!‚(:à ¹·æÔ↮€ disk.drbd2080.out‚0& disk.nvme0c0n1.out‚28282.849609375 network.recv‚ 2.451884e+06) disk./.usagePercent‚20.31788972661836 disk.sda1.in‚0 disk.sda3.in‚ 0.01171875 cpu‚25.355580682962113 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0$ memory_percent‚93.53272914886475 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0* proc.memory.percent‚16.001558303833008 network.sent‚ 2.1012576e+07$ disk.nvme0n1.out‚28282.849609375 disk.rbd0.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.sda3.out‚ 264.265625) proc.memory.availableMB‚ 1059.59765625" proc.memory.rssMB‚ 2621.6953125$ disk./.usageGB‚302.7512397766113 disk.dm-0.in‚0.078125! disk.dm-0.out‚28282.849609375 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda.out‚ 264.265625 disk.rbd1.in‚0 disk.rbd1.out‚ 37.228515625 disk.drbd1716.in‚0 proc.cpu.threads‚17 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.out‚ 264.265625°5ñˆ2à!- _wandbruntime‚6885  _runtime‚6885ÄgUáýá!êM E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚ 6885.40809475 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚41492185 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚557672469218816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 163453605 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 289499129 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 163473669 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚503769094260736 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 126025458 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 126025460 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7226203999434 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚73535332 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7252909085696 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 115027078 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚557672652575232 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 289479063 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 126025458 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚73535334 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚550064361377792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 % _timestamp‚1.7884517699358428e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚503769076257792 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚490555864797184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7252252709888 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚550064385317888 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚497782085835978 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 163473669 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.sm¿€ ©óall_pool.current‚ 656375808 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7252252709888 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7252909085696 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 163453605 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 trainer/global_step‚8649 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚503769094260736 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚497781414358654 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 289499129 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 115027519 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7608267257344 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 289479063 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚490555881836544 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚511022003346432ï÷yIõ&â!ïM KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚557672652575232 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 289479063 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 126025458 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚73535334 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚550064361377792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 % _timestamp‚1.7884517699358428e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚503769076257792 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚490555864797184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7252252709888 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚439 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚550064385317888 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚497782085835978 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 163473669 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7252252709888 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7252909085696 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 163453605 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 trainer/global_step‚8649 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚503769094260736 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚497781414358654 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 289499129 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 115027519 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7608267257344 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 289479063 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚490555881836544 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚511022003346432 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚ 6885.40809475 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚41492185 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚557672469218816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 163453605 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 289499129 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 163473669 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚503769094260736 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 126025458 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 126025460 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7226203999434 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚73535332 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7252909085696 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 115027078ÇÄBhÂ2ã!- _wandbruntime‚6885  _runtime‚6885Y\Å]ä!× 1loss/FBt3/assignment_loss‚0.18914158198782216 0loss/FBt4/assignment_loss‚0.6107094836673544 1loss/SRqqt2/detection_loss‚1.9078616494057226 0loss/FRt4/assignment_loss‚2.9256463756163913 /loss/FBt3/detection_loss‚0.5595590218495673 /loss/FBt4/detection_loss‚3.5731107791264853 trainer/global_step‚8649 _step‚712 2loss/SRqqt2/assignment_loss‚1.3817926152956252 1loss/FBt2/assignment_loss‚0.33745600318069746 /loss/FBt1/detection_loss‚2.9629463390870527 /loss/FBt2/detection_loss‚2.4925102312117815 _runtime‚6885.409447874 0loss/FRt3/assignment_loss‚1.5185985998673872 2loss/SRqqt3/assignment_loss‚1.1443759082343246 1loss/FBt1/assignment_loss‚0.44108230484355526 %loss/total_loss‚46.31856234358369 /loss/FRt1/detection_loss‚3.1926531224023726 /loss/FRt3/detection_loss‚1.6395197594707662 0loss/SRqqt1/detection_loss‚2.747731016440825 1loss/SRqqt4/detection_loss‚1.4872188966786652 /loss/FRt2/assignment_loss‚3.139429553252895 /loss/FRt2/detection_loss‚2.2113287296237014 .loss/FRt4/detection_loss‚2.124415842195352 1loss/SRqqt3/detection_loss‚1.2152120334203127 epoch‚7 /loss/FRt1/assignment_loss‚4.702154914538066 1loss/SRqqt1/assignment_loss‚2.290703980760141 2loss/SRqqt4/assignment_loss‚1.5234036014268273 % _timestamp‚1.7884517699376209e+09U9Z$bå!Ü 2loss/SRqqt2/assignment_loss‚1.3817926152956252 1loss/FBt2/assignment_loss‚0.33745600318069746 /loss/FBt1/detection_loss‚2.9629463390870527 /loss/FBt2/detection_loss‚2.4925102312117815 _runtime‚6885.409447874 0loss/FRt3/assignment_loss‚1.5185985998673872 2loss/SRqqt3/assignment_loss‚1.1443759082343246 1loss/FBt1/assignment_loss‚0.44108230484355526 %loss/total_loss‚46.31856234358369 /loss/FRt1/detection_loss‚3.1926531224023726 /loss/FRt3/detection_loss‚1.6395197594707662 0loss/SRqqt1/detection_loss‚2.747731016440825 1loss/SRqqt4/detection_loss‚1.4872188966786652 /loss/FRt2/assignment_loss‚3.139429553252895 /loss/FRt2/detection_loss‚2.2113287296237014 .loss/FRt4/detection_loss‚2.124415842195352 1loss/SRqqt3/detection_loss‚1.2152120334203127 epoch‚7 /loss/FRt1/assignment_loss‚4.702154914538066 1loss/SRqqt1/assignment_loss‚2.290703980760141 2loss/SRqqt4/assignment_loss‚1.5234036014268273 % _timestamp‚1.7884517699376209e+09 1loss/FBt3/assignment_loss‚0.18914158198782216 0loss/FBt4/assignment_loss‚0.6107094836673544 1loss/SRqqt2/detection_loss‚1.9078616494057226 0loss/FRt4/assignment_loss‚2.9256463756163913 /loss/FBt3/detection_loss‚0.5595590218495673 /loss/FBt4/detection_loss‚3.5731107791264853 trainer/global_step‚8649 _step‚712È×£…x“æ!‚(:ˆ È·æÔøÍòý gpu.0.gpu‚48.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.482' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8964 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ØÚuç!‚(:ê È·æÔ¡³›ÿ disk.drbd2265.out‚0 disk.drbd2080.out‚0* proc.memory.percent‚16.001558303833008 network.recv‚ 2.458848e+06 disk.drbd1716.in‚0 disk.rbd0.out‚0 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125' disk.nvme0c0n1.out‚28326.7041015625 disk.dm-0.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.rbd1.in‚0 disk.rbd1.out‚ 37.3232421875* disk./.usagePercent‚20.318254025339584 disk.sda.out‚ 265.00390625 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.out‚ 265.00390625% disk.nvme0n1.out‚28326.7041015625 disk.drbd2266.out‚0 network.sent‚ 2.1110213e+07$ disk./.usageGB‚302.7566680908203 disk.dm-1.in‚ 0.01171875 disk.rbd0.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.sda.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 265.00390625" disk.dm-0.out‚28326.7041015625 cpu‚25.354927836291868 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.sda2.out‚0# memory_percent‚93.5349702835083) proc.memory.availableMB‚ 1059.23046875 disk.sda1.in‚0&R“è!‚(:ˆ ×·æÔû…ƒý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚176.524' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.3048 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0õÐZ•oé!‚(:ä ×·æÔ“¤Œÿ disk.sda2.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875) disk./.usagePercent‚20.31876680843067 disk.dm-1.out‚ 265.40625 disk.drbd1716.in‚0 cpu‚25.354270083708414 disk.sda3.in‚ 0.01171875% disk./.usageGB‚302.76430892944336 disk.drbd2080.out‚0 disk.drbd2266.out‚0* proc.memory.percent‚16.001558303833008 disk.nvme0n1.in‚0.078125 disk.rbd0.out‚0 disk.rbd1.out‚ 37.4482421875" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚28366.53369140625 disk.rbd0.in‚0# disk.dm-0.out‚28366.53369140625 disk.sda.in‚ 0.01171875 disk.sda1.out‚0& disk.nvme0n1.out‚28366.53369140625 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0# memory_percent‚93.5319185256958) proc.memory.availableMB‚ 1059.73046875 disk.drbd2265.out‚0 disk.sda.out‚ 265.40625 network.sent‚ 2.1207242e+07 network.recv‚ 2.463834e+06 disk.drbd2265.in‚0 disk.sda3.out‚ 265.40625 disk.sda1.in‚0_Ö¤¬2ê!- _wandbruntime‚6923  _runtime‚6923 hÂ\Éë!à trainer/global_step‚8699 % _timestamp‚1.7884518078595543e+09 _runtime‚6923.331170491 _step‚713 ' lr-AdamW/pg1‚2.8009964683768623e-05 ' lr-AdamW/pg2‚2.8009964683768623e-05¾µê©Îì!È ' lr-AdamW/pg2‚2.8009964683768623e-05 trainer/global_step‚8699 % _timestamp‚1.7884518078595543e+09 _runtime‚6923.331170491 _step‚713 ' lr-AdamW/pg1‚2.8009964683768623e-05ɤζ2í!- _wandbruntime‚6923  _runtime‚6923jRJ¦ç'î!áO =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚506530393164288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41712843 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚553085271463424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚500512407137654 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚500511735660538 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 164361203 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚553085247523328 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7293075450368 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚41713266 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚8699 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚513822794249216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7650209841152 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7266222903670 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 126712242 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 _runtime‚6923.332163956 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚513822794249216 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 291093485 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 164381241 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚506530393164288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚425 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚423 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 126712242 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7293075450368 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚506530375161344 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 126712244 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 291073445 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 115648832 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2¡ÞöÒï!æO O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7293075450368 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚41713266 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚8699 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚513822794249216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7650209841152 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7266222903670 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 126712242 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 _runtime‚6923.332163956 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚513822794249216 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 291093485 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 164381241 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚506530393164288 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚425 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚423 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 126712242 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7293075450368 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚506530375161344 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 126712244 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 291073445 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 115648832 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚506530393164288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41712843 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚553085271463424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚500512407137654 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚500511735660538 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 164361203 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚553085247523328 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0Êk /2ð!- _wandbruntime‚6924  _runtime‚6924«¹zšð&ñ!êM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7266384900878 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 164399721 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚506586746203136 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 291105915 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚506586746203136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚73943732 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚41717317 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _runtime‚ 6924.06021359 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚513880641094656 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7293238515712 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚513879966715904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7293894891520 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 291125981 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 115661482 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7651234402816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7267039338842 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚73943734 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 291125981 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7293238515712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚433 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 126726258 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 164379657 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚41717748 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚493301088364544 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7293894891520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 115661049 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7884518085881913e+09 _step‚715 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 291105915 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚500567456226062 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 126726260 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 126726258 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 164399721 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚513880641094656 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚506586728200192 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚553146922201088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚553146898260992Z û’õ&ò!ïM P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 126726260 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚560798156603904 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚560797973247488 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 291105915 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚506586746203136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚73943732 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚41717317 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _runtime‚ 6924.06021359 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚513880641094656 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7293238515712 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚513879966715904 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7293894891520 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 291125981 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 115661482 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7651234402816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7267039338842 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚73943734 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 291125981 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7293238515712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚433 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 126726258 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 164379657 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚41717748 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚493301088364544 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7293894891520 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 115661049 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7884518085881913e+09 _step‚715 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 291105915 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚500567456226062 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 126726260 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 126726258 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 164399721 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚513880641094656 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚506586728200192 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚553146922201088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚553146898260992 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7266384900878 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 164399721 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚506586746203136 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚516682106362880 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7306403805322 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚516681431997440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚41939744 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7333404893696 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 165307293 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7334061256192 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚74343964 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 165287255 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7693406064128 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 116283265 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 127413042 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚509348027103744 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 127413042 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚563861214410752 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 292720337 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚556167808346624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚509348045106688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7333404893696 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41939301 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚516682106362880 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 127413044 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚74343964 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 165287255 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7693406064128 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 116283265 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 127413042 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚509348027103744 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 127413042 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚443 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚563861214410752 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 292720337 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚556167808346624 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚509348045106688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7333404893696 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚41939301 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚516682106362880 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 127413044 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚516682106362880 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7306403805322 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚516681431997440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚41939744 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7333404893696 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 165307293 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7334061256192 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0ÎÆÊ2‚"- _wandbruntime‚6962  _runtime‚6962w›ÞN"ƒ"ëM OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚563923729804288 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚8749 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _step‚719 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚563923552739328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 292732767 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚509404398145536 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 127427060 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 292732767 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚496046294892544 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 116296342 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚74352132 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚74352134 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7334880697344 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7334880697344 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚516739278842880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚496046277853184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚556229459084288 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚516738604464128 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 165325773 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7307220240286 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7307874678250 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 292752833 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7694270720000 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 165325773 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚433 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚556229435144192 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚503354169570794 KEDeviceStatsMonitorÈk£.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 165305709 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 292752833 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 127427060 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7334224321536 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚41943777 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 116295909 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 _runtime‚6962.259029778×0ß™ö&„"ðM OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚509404380142592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚509404380142592 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7307874678250 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 292752833 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7694270720000 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 165325773 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚433 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚556229435144192 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚503354169570794 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 165305709 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 292752833 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 127427060 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7334224321536 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚41943777 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 116295909 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 _runtime‚6962.259029778 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚563923729804288 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚8749 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 _step‚719 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚563923552739328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 292732767 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚509404398145536 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 127427060 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 292732767 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚496046294892544 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 116296342 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚74352132 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚74352134 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7334880697344 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7334880697344 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚516739278842880 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚496046277853184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚556229459084288 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚516738604464128 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 165325773 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7307220240286 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚74752366 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚506084490872470 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚519540744111104 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚42164180 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 174981120 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 116916544 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚519540069745664 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 294347189 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7884518846961923e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 294327149 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚722 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 166213307 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚506083819395354 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚7000.168124876 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 128113842 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7375047062016 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7736236914688 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚519540069745664 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 128113842 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚566986558204416 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7375047062016 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 166213307 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7736387955712 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚566986558204416 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7375047062016 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 166213307 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7736387955712 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚74752366 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚506084490872470 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚519540744111104 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚42164180 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 174981120 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 116916544 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚519540069745664 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 294347189 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 % _timestamp‚1.7884518846961923e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 294327149 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚722 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 166213307 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚429 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚506083819395354 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚7000.168124876 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 128113842 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7375047062016 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7736236914688 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944Ò¥kT2’"- _wandbruntime‚7000  _runtime‚7000®Xð&“"êM F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚723 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 116929240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 128127860 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 128127860 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7737268050432 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 294379685 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7348055579694 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚512222032084992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚559311972027392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚512222050087936 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 116929687 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚74760532 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚567049086952960 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7737114925568 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7375210127360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚498791484381184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚42169153 trainer/global_step‚8799 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 294359619 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7375210127360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚506139539960878 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 166251825 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚519597916591104 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚506140211438202 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7375866503168 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 166251825 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚42168708 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7348710017658 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7375866503168 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 294379685 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚74760534 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 $ _timestamp‚1.788451885419927e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0´Ù×õ&”"ïM J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7737268050432 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 294379685 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7348055579694 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚512222032084992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚559311972027392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚512222050087936 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 116929687 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚74760532 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚567049086952960 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7737114925568 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7375210127360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚498791484381184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚42169153 trainer/global_step‚8799 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 294359619 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7375210127360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚506139539960878 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 166251825 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚519597916591104 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚506140211438202 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7375866503168 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 166251825 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚42168708 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7348710017658 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7375866503168 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 294379685 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚74760534 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 $ _timestamp‚1.788451885419927e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚723 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 116929240 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 128127860 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 128127860 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944Ó^ÝN2•"- _wandbruntime‚7000  _runtime‚7000ž¤Ûï[–"Õ /loss/FRt1/assignment_loss‚3.996932375815607 /loss/FRt3/detection_loss‚1.7433649660260588 /loss/FBt4/detection_loss‚3.4906447331110635 %loss/total_loss‚44.81331623757542 epoch‚7 2loss/SRqqt2/assignment_loss‚1.3295013045147062 1loss/SRqqt2/detection_loss‚1.6089156310260295 % _timestamp‚1.7884518854214768e+09 _step‚724 0loss/FBt4/assignment_loss‚0.6990253121158881 0loss/SRqqt1/detection_loss‚2.596061670780182 /loss/FBt2/detection_loss‚2.3676319320996604 _runtime‚7000.893045014 1loss/FBt1/assignment_loss‚0.24352623742045693 0loss/FBt2/assignment_loss‚0.3756762352145529 /loss/FRt4/detection_loss‚2.2940034762672754 1loss/SRqqt4/detection_loss‚1.5503780751178662 /loss/FBt3/detection_loss‚0.6758115636414362 trainer/global_step‚8799 1loss/SRqqt3/assignment_loss‚1.128225785873515 0loss/FBt3/assignment_loss‚0.2913373693016517 .loss/FRt2/detection_loss‚2.173837067640346 0loss/FRt3/assignment_loss‚1.5987296038203769 2loss/SRqqt4/assignment_loss‚1.4357601882268984 .loss/FRt1/detection_loss‚2.895691310205767 /loss/FBt1/detection_loss‚3.3356383979320525 0loss/FRt4/assignment_loss‚2.6517642282921337 2loss/SRqqt1/assignment_loss‚2.1006877263387045 0loss/FRt2/assignment_loss‚3.0441347567931465 1loss/SRqqt3/detection_loss‚1.1860362900000425ì‘h!`—"Ú 0loss/FBt4/assignment_loss‚0.6990253121158881 0loss/SRqqt1/detection_loss‚2.596061670780182 /loss/FBt2/detection_loss‚2.3676319320996604 _runtime‚7000.893045014 1loss/FBt1/assignment_loss‚0.24352623742045693 0loss/FBt2/assignment_loss‚0.3756762352145529 /loss/FRt4/detection_loss‚2.2940034762672754 1loss/SRqqt4/detection_loss‚1.5503780751178662 /loss/FBt3/detection_loss‚0.6758115636414362 trainer/global_step‚8799 1loss/SRqqt3/assignment_loss‚1.128225785873515 0loss/FBt3/assignment_loss‚0.2913373693016517 .loss/FRt2/detection_loss‚2.173837067640346 0loss/FRt3/assignment_loss‚1.5987296038203769 2loss/SRqqt4/assignment_loss‚1.4357601882268984 .loss/FRt1/detection_loss‚2.895691310205767 /loss/FBt1/detection_loss‚3.3356383979320525 0loss/FRt4/assignment_loss‚2.6517642282921337 2loss/SRqqt1/assignment_loss‚2.1006877263387045 0loss/FRt2/assignment_loss‚3.0441347567931465 1loss/SRqqt3/detection_loss‚1.1860362900000425 /loss/FRt1/assignment_loss‚3.996932375815607 /loss/FRt3/detection_loss‚1.7433649660260588 /loss/FBt4/detection_loss‚3.4906447331110635 %loss/total_loss‚44.81331623757542 epoch‚7 2loss/SRqqt2/assignment_loss‚1.3295013045147062 1loss/SRqqt2/detection_loss‚1.6089156310260295 % _timestamp‚1.7884518854214768e+09 _step‚724Ô#€ˆM’˜"‚(:‡ ±¸æÔ§Ôþ gpu.0.gpu‚9.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚123.797' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.7594 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ng0Ït™"‚(:é ±¸æÔ¹êÒ disk.rbd0.in‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚28584.92333984375 disk.sda.in‚ 0.01171875 disk.sda.out‚ 268.6796875 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 cpu‚25.35032235520931 network.sent‚ 2.144802e+07% disk./.usageGB‚302.79614639282227 disk.sda1.out‚0 disk.drbd2080.out‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125( disk.nvme0c0n1.out‚28584.92333984375* disk./.usagePercent‚20.320903447311437 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125* proc.memory.percent‚16.001558303833008 network.recv‚ 2.493243e+06 disk.sda1.in‚0 disk.rbd1.out‚ 38.1044921875 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 268.6796875 disk.sda3.out‚ 268.6796875 disk.drbd2266.in‚0( proc.memory.availableMB‚ 1059.1953125& disk.nvme0n1.out‚28584.92333984375 disk.drbd1716.in‚0 disk.rbd0.out‚0$ memory_percent‚93.53518486022949" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.drbd2266.out‚0mÐ;“š"‚(:ˆ À¸æÔ»¾šý gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.766' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5532 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0!Ó=âk›"‚(:à À¸æÔüƒÿ network.sent‚ 2.1557047e+07) disk./.usagePercent‚20.32256928833674$ disk./.usageGB‚302.8209686279297 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.out‚ 28633.5546875 disk.rbd1.out‚ 38.1669921875 disk.drbd1716.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125$ disk.nvme0c0n1.out‚ 28633.5546875 disk.sda3.in‚ 0.01171875" proc.memory.rssMB‚ 2621.6953125 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 269.48046875" disk.nvme0n1.out‚ 28633.5546875 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 network.recv‚ 2.500265e+06 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0$ memory_percent‚93.53363513946533) proc.memory.availableMB‚ 1059.44921875 cpu‚25.34970219714631 proc.cpu.threads‚17 disk.dm-1.out‚ 269.48046875 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda3.out‚ 269.48046875 disk.drbd2266.in‚0* proc.memory.percent‚16.0015583038330089WF“œ"‚(:ˆ ϸæÔ’ ëý gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.101' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0202 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÇLôj@"‚(:è ϸæÔðá„€ disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0% proc.memory.availableMB‚ 1059.6875* proc.memory.percent‚16.001558303833008% disk./.usageGB‚302.82648849487305 disk.drbd2265.out‚0 disk.drbd2266.out‚0" disk.dm-0.out‚28673.6103515625$ memory_percent‚93.53218078613281 proc.cpu.threads‚17* disk./.usagePercent‚20.322939731238783 disk.rbd1.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875% disk.nvme0n1.out‚28673.6103515625 disk.drbd1716.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.sda3.out‚ 269.80859375 network.recv‚ 2.503814e+06 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.sda2.out‚0 disk.rbd1.out‚ 38.3623046875 cpu‚25.349058951527166 disk.drbd2080.in‚0 disk.sda.out‚ 269.80859375 disk.dm-1.out‚ 269.80859375 network.sent‚ 2.1564405e+07 disk.drbd2080.out‚0' disk.n(ñöú3vme0c0n1.out‚28673.6103515625 disk.rbd0.in‚0”HLf2ž"- _wandbruntime‚7038  _runtime‚7038_p¥ÉŸ"à trainer/global_step‚8849 % _timestamp‚1.7884519232521112e+09 _runtime‚7038.723722558 _step‚725 ' lr-AdamW/pg1‚2.5766960099610166e-05 ' lr-AdamW/pg2‚2.5766960099610166e-051x޶Π"È ' lr-AdamW/pg1‚2.5766960099610166e-05 ' lr-AdamW/pg2‚2.5766960099610166e-05 trainer/global_step‚8849 % _timestamp‚1.7884519232521112e+09 _runtime‚7038.723722558 _step‚725Õ·†p¯2¡"- _wandbruntime‚7038  _runtime‚7038]ðç'¢"áO ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7388074484138 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 117552352 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 117551917 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7416032867840 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚514983330988544 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7415376505344 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 128814642 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 128814642 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚435 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7416032867840 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚570112115265024 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 128814644 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚514983348991488 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7388728921894 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚514983330988544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7415376505344 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 167159397 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚433 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 167159397 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚562332858172928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 295954001 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚562332882113024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚522398707493888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 167139359 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚522398707493888 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚42391586 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚514983348991488 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7779418618880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚501481803817984 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 128814644 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚42391153 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚75160766 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7779257092096 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚522399381859328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚501481786778624‘x©xì'£"æO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7416032867840 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚514983330988544 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7415376505344 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 128814642 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 128814642 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚435 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7416032867840 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚570112115265024 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 128814644 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚514983348991488 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7388728921894 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚514983330988544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7415376505344 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 167159397 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚433 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 167159397 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚562332858172928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 295954001 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚562332882113024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚522398707493888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 167139359 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚522398707493888 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚42391586 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚514983348991488 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7779418618880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚501481803817984 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 128814644 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚42391153 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚75160766 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7779257092096 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚522399381859328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚501481786778624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7388074484138 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 117552352 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 117551917 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872Öû»b2¤"- _wandbruntime‚7039  _runtime‚7039‚¼ý«ñ&¥"ëM ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚515039702030336 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚75168934 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7780282127872 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 trainer/global_step‚8849 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7416852308992 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚75168932 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚501536707948544 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 117564595 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚570174631622144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚508925581828286 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 128828658 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 117565030 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7416195933184 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 296006537 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚522456554339328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚42396096 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚562394508910592 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚508926253305610 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚501536690909184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884519239722862e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚522456554339328 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 167157813 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 167177877 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 _step‚727 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7416852308992 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚7039.444314833 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚522455879960576 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 296006537 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 167157813 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7389545357066 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚515039702030336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚562394532850688 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844ñvZ–¦"ðM I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 296006537 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚522456554339328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚42396096 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚562394508910592 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚508926253305610 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚501536690909184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884519239722862e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚522456554339328 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 167157813 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 167177877 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 _step‚727 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7416852308992 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚7039.444314833 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚522455879960576 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 296006537 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 167157813 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7389545357066 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚515039702030336 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚562394532850688 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚515039702030336 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚75168934 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7780282127872 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 trainer/global_step‚8849 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7416852308992 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚75168932 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚501536707948544 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 117564595 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚570174631622144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚508925581828286 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 128828658 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 117565030 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7416195933184 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944×c"(2§"- _wandbruntime‚7039  _runtime‚7039”ëEX¨"Ò 1loss/SRqqt1/assignment_loss‚2.235545476898551 1loss/SRqqt3/detection_loss‚1.0322388731874526 1loss/SRqqt4/detection_loss‚1.4820627852032582 .loss/FBt1/detection_loss‚2.493208736181259 % _timestamp‚1.7884519239740286e+09 0loss/FRt4/assignment_loss‚2.7497390757907523 2loss/SRqqt2/assignment_loss‚1.5583391869590888 0loss/SRqqt1/detection_loss‚2.509106084704399 0loss/SRqqt2/detection_loss‚1.599749289693371 _step‚728 %loss/total_loss‚42.15413603030274 0loss/FRt1/assignment_loss‚2.9756094068288803 1loss/FBt2/assignment_loss‚0.27921394430673985 .loss/FRt1/detection_loss‚3.109117570022742 .loss/FRt4/detection_loss‚2.260778905857693 /loss/FBt2/detection_loss‚1.9427456649450154 trainer/global_step‚8849 /loss/FRt2/detection_loss‚2.0571952178034674 /loss/FRt2/assignment_loss‚3.403141147868578 0loss/FRt3/assignment_loss‚1.7663427528106805 2loss/SRqqt3/assignment_loss‚0.9611231889284682 2loss/SRqqt4/assignment_loss‚1.2155917406703034 /loss/FBt4/assignment_loss‚0.444844189286232 epoch‚7 .loss/FBt3/detection_loss‚0.717991105462243 /loss/FBt4/detection_loss‚3.1243399024009704 0loss/FBt1/assignment_loss‚0.5110868646071793 1loss/FBt3/assignment_loss‚0.19463978903836932 /loss/FRt3/detection_loss‚1.5303851308470422 _runtime‚7039.445539996iñ‹R]©"× 0loss/FRt4/assignment_loss‚2.7497390757907523 2loss/SRqqt2/assignment_loss‚1.5583391869590888 0loss/SRqqt1/detection_loss‚2.509106084704399 0loss/SRqqt2/detection_loss‚1.599749289693371 _step‚728 %loss/total_loss‚42.15413603030274 0loss/FRt1/assignment_loss‚2.9756094068288803 1loss/FBt2/assignment_loss‚0.27921394430673985 .loss/FRt1/detection_loss‚3.109117570022742 .loss/FRt4/detection_loss‚2.260778905857693 /loss/FBt2/detection_loss‚1.9427456649450154 trainer/global_step‚8849 /loss/FRt2/detection_loss‚2.0571952178034674 /loss/FRt2/assignment_loss‚3.403141147868578 0loss/FRt3/assignment_loss‚1.7663427528106805 2loss/SRqqt3/assignment_loss‚0.9611231889284682 2loss/SRqqt4/assignment_loss‚1.2155917406703034 /loss/FBt4/assignment_loss‚0.444844189286232 epoch‚7 .loss/FBt3/detection_loss‚0.717991105462243 /loss/FBt4/detection_loss‚3.1243399024009704 0loss/FBt1/assignment_loss‚0.5110868646071793 1loss/FBt3/assignment_loss‚0.19463978903836932 /loss/FRt3/detection_loss‚1.5303851308470422 _runtime‚7039.445539996 1loss/SRqqt1/assignment_loss‚2.235545476898551 1loss/SRqqt3/detection_loss‚1.0322388731874526 1loss/SRqqt4/detection_loss‚1.4820627852032582 .loss/FBt1/detection_loss‚2.493208736181259 % _timestamp‚1.7884519239740286e+09Ø'‰N‘ª"‚(:† Þ¸æÔîø©ý gpu.0.gpu‚38.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚171.82' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.364 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¢Yõ`x«"‚(:í Þ¸æÔ·¶×ÿ* disk./.usagePercent‚20.323480675158635 disk.dm-0.in‚0.078125 disk.dm-1.in‚ 0.01171875 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 38.4248046875) proc.memory.availableMB‚ 1059.37890625" proc.memory.rssMB‚ 2621.6953125 disk.sda.in‚ 0.01171875 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 270.51171875 disk.drbd2265.in‚0$ memory_percent‚93.53406429290771 network.sent‚ 2.156896e+07$ disk./.usageGB‚302.8345489501953 disk.sda3.out‚ 270.51171875 cpu‚25.348427934668464 network.recv‚ 2.508801e+06 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda.out‚ 270.51171875& disk.nvme0n1.out‚28718.43212890625 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚28718.43212890625 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.drbd1716.out‚0# disk.dm-0.out‚28718.43212890625 disk.sda2.in‚0 disk.drbd2265.out‚0* proc.memory.percent‚16.001558303833008ýøR|“¬"‚(:ˆ í¸æÔ¦ñ±ý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚166.131' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚33.2262 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0àÃv­"‚(:ë í¸æԪɮÿ network.recv‚ 2.514389e+06 disk.sda.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125" proc.memory.rssMB‚ 2621.6953125 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 cpu‚25.34776411396334 disk.sda.out‚ 270.95703125) disk./.usagePercent‚20.32591402677159$ disk./.usageGB‚302.8708076477051 disk.rbd0.out‚0 proc.cpu.threads‚17# disk.dm-0.out‚28757.22998046875 disk.sda1.out‚0( disk.nvme0c0n1.out‚28757.22998046875* proc.memory.percent‚16.001558303833008 disk.rbd1.in‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 270.95703125 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 38.4248046875 disk.drbd1716.out‚0( proc.memory.availableMB‚ 1059.1328125 network.sent‚ 2.1676706e+07 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125& disk.nvme0n1.out‚28757.22998046875 disk.dm-1.out‚ 270.95703125 disk.drbd2080.out‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd2265.out‚0$ memory_percent‚93.53556632995605]†HŽ2®"- _wandbruntime‚7077  _runtime‚7077îÖWîů"¿ trainer/global_step‚8899 # _timestamp‚1.78845196154366e+09 _runtime‚7077.015269171 _step‚729 & lr-AdamW/pg1‚2.503474725642195e-05 & lr-AdamW/pg2‚2.503474725642195e-05ÌH¦´Ê°"Ä & lr-AdamW/pg1‚2.503474725642195e-05 & lr-AdamW/pg2‚2.503474725642195e-05 trainer/global_step‚8899 # _timestamp‚1.78845196154366e+09 _runtime‚7077.015269171 _step‚729ÙÅöz­2±"- _wandbruntime‚7077  _runtime‚7077×þÉç'²"áO WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7456362311168 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 129515444 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 297580853 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 168085449 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚525257345242112 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7457018673664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 118187936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚565415418996224 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚517801000933888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 129515444 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚525258019607552 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7822223965184 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7456362311168 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 168065411 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚75569166 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚730 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7429564261302 % _timestamp‚1.7884519615443902e+09 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 129515442 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 168085449 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7428909823546 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7457018673664 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚517801000933888 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚75569164 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚511656574607286 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 129515442 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚573237619021312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 168065411 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚42618772 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚525258019607552 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚565415395056128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚8899 _runtime‚7077.016206902 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚504227010345984 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚517801000933888 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚75569164 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚511656574607286 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 129515442 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚573237619021312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 168065411 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚42618772 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚525258019607552 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚565415395056128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚8899 _runtime‚7077.016206902 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚504227010345984 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚525257345242112 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7457018673664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 118187936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚565415418996224 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚517801000933888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_baSÓ> tch_start/active.large_pool.allocated‚ 129515444 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚525258019607552 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7822223965184 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7456362311168 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 168065411 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚75569166 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚730 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7429564261302 % _timestamp‚1.7884519615443902e+09 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 129515442 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 168085449 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7428909823546 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7457018673664 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430ÚYoÑr2´"- _wandbruntime‚7077  _runtime‚7077$« 5ð&µ"êM H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 297633389 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7457838114816 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚517857335969792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 297613323 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 168083865 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚511711623695694 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 118200653 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚42623321 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚511712295173018 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 118201108 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚75577332 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 168103929 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 168083865 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 $ _timestamp‚1.788451962273187e+09 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚517857353972736 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚565477069733888 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚565477045793792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚7077.745210937 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚8899 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 129529458 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 129529460 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚731 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 297633389 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324²Ñ Çõ&¶"ïM D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚731 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 297633389 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 297633389 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7457838114816 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚517857335969792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 297613323 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 168083865 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚511711623695694 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 118200653 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚42623321 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚511712295173018 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 118201108 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚75577332 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 168103929 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 168083865 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 $ _timestamp‚1.788451962273187e+09 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚517857353972736 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚565477069733888 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚565477045793792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚7077.745210937 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚8899 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 129529458 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 129529460 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚506972199834624 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 130216242 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 168991463 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bܦ·ytes.all.peak‚ 337538560 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚568497931939328 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚528115982990336 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7498004479488 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 299227745 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚514441944997578 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7498004479488 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚75977566 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 130216244 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚7115.704984943 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7865100852736 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 _step‚734 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚528115982990336 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 130216242 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7497348116992 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 118823613 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 trainer/global_step‚8949 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 130216244 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 118824066 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚576363032792064 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚568497955879424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚528116657355776 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7497348116992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚528116657355776 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚520618634873344 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚75977564 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚506972216873984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚576363218258944 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7470399600710 % _timestamp‚1.7884520002331307e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚528115982990336 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 130216242 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7497348116992 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 118823613 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 trainer/global_step‚8949 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 130216244 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 118824066 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚576363032792064 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚568497955879424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚528116657355776 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7497348116992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚528116657355776 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚520618634873344 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚75977564 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚506972216873984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚576363218258944 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7470399600710 % _timestamp‚1.7884520002331307e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚506972199834624 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 130216242 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 168991463 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚568497931939328 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚528115982990336 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7498004479488 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 299227745 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚514441944997578 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7498004479488 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚75977566 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 130216244 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚7115.704984943 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7865100852736Þ¨‰ n2Æ"- _wandbruntime‚7116  _runtime‚7116kú˜Žñ&Ç"ëM WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 130230260 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7498823920640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7470561597918 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚520674987912192 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7865950359552 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 299260241 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚528173829835776 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7498823920640 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚507027121004544 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 299240175 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 118836301 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚568559606617088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7471216035882 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚520674987912192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚8949 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 169009917 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7498167544832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 299260241 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7866122358784 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884520009488747e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 130230258 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚528173155457024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 _runtime‚7116.421148072 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 130230260 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚42851024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚514497665563102 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 169029981 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚568559582676992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚75985732 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚735 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 169009917 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚576425728975872 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚576425533036544±oÈ"ðM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884520009488747e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 130230258 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚528173155457024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 _runtime‚7116.421148072 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 130230260 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚42851024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚514497665563102 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 169029981 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚568559582676992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚75985732 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStaÓ§—mßtsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚735 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 169009917 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚576425728975872 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚576425533036544 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 130230260 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7498823920640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7470561597918 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚520674987912192 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7865950359552 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 299260241 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚528173829835776 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7498823920640 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚507027121004544 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 299240175 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 118836301 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚568559606617088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7471216035882 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚520674987912192 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 trainer/global_step‚8949 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 169009917 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7498167544832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 299260241 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7866122358784 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0ßM%­Õ2É"- _wandbruntime‚7116  _runtime‚7116o¡uYÊ"Ó 2loss/SRqqt2/assignment_loss‚1.5635625248731568 1loss/FBt1/assignment_loss‚0.24749245212205673 trainer/global_step‚8949 % _timestamp‚1.7884520009507625e+09 .loss/FRt1/detection_loss‚3.112814328887246 .loss/FRt4/detection_loss‚1.903549636563947 1loss/SRqqt2/detection_loss‚1.9233269823151966 1loss/SRqqt3/detection_loss‚1.0417884353181006 1loss/SRqqt4/detection_loss‚1.6099100817115075 /loss/FRt2/assignment_loss‚3.831989078418068 /loss/FRt3/assignment_loss‚2.179624537626902 2loss/SRqqt3/assignment_loss‚1.3820074235784336 %loss/total_loss‚44.62180906603183 2loss/SRqqt1/assignment_loss‚2.2004641554294486 0loss/SRqqt1/detection_loss‚2.215773541193742 /loss/FBt1/detection_loss‚2.7931286692619324 _step‚736 /loss/FRt4/assignment_loss‚2.639260647758361 -loss/FRt3/detection_loss‚1.99232842742155 /loss/FBt2/detection_loss‚2.5503715525070825 /loss/FBt3/detection_loss‚0.7740010910201818 epoch‚7 /loss/FRt1/assignment_loss‚3.211088256402449 .loss/FBt4/detection_loss‚3.326627418398857 2loss/SRqqt4/assignment_loss‚1.3006468051744084 1loss/FBt2/assignment_loss‚0.48723453609272366 1loss/FBt3/assignment_loss‚0.18428714126809195 1loss/FBt4/assignment_loss‚0.06558489170856774 /loss/FRt2/detection_loss‚2.0849464509798135 _runtime‚7116.422287784™½‹^Ë"Ø 2loss/SRqqt4/assignment_loss‚1.3006468051744084 1loss/FBt2/assignment_loss‚0.48723453609272366 1loss/FBt3/assignment_loss‚0.18428714126809195 1loss/FBt4/assignment_loss‚0.06558489170856774 /loss/FRt2/detection_loss‚2.0849464509798135 _runtime‚7116.422287784 2loss/SRqqt2/assignment_loss‚1.5635625248731568 1loss/FBt1/assignment_loss‚0.24749245212205673 trainer/global_step‚8949 % _timestamp‚1.7884520009507625e+09 .loss/FRt1/detection_loss‚3.112814328887246 .loss/FRt4/detection_loss‚1.903549636563947 1loss/SRqqt2/detection_loss‚1.9233269823151966 1loss/SRqqt3/detection_loss‚1.0417884353181006 1loss/SRqqt4/detection_loss‚1.6099100817115075 /loss/FRt2/assignment_loss‚3.831989078418068 /loss/FRt3/assignment_loss‚2.179624537626902 2loss/SRqqt3/assignment_loss‚1.3820074235784336 %loss/total_loss‚44.62180906603183 2loss/SRqqt1/assignment_loss‚2.2004641554294486 0loss/SRqqt1/detection_loss‚2.215773541193742 /loss/FBt1/detection_loss‚2.7931286692619324 _step‚736 /loss/FRt4/assignment_loss‚2.639260647758361 -loss/FRt3/detection_loss‚1.99232842742155 /loss/FBt2/detection_loss‚2.5503715525070825 /loss/FBt3/detection_loss‚0.7740010910201818 epoch‚7 /loss/FRt1/assignment_loss‚3.211088256402449 .loss/FBt4/detection_loss‚3.326627418398857àMÈ÷IžÌ"‚(:“ ©¹æÔðáûü gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚147.588' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.517599999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0]]ßRkÍ"‚(:à ©¹æÔËÔ€ disk.drbd1716.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0& proc.memory.availableMB‚ 1059.28125* proc.memory.percent‚16.001558303833008 proc.cpu.threads‚17) disk./.usagePercent‚20.32834584233934 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 network.recv‚ 2.532831e+06 disk.sda2.out‚0& disk.nvme0n1.out‚28917.44287109375 disk.dm-0.in‚0.078125 disk.sda.out‚ 272.96875 cpu‚25.345262144338406% disk./.usageGB‚302.90704345703125 disk.rbd1.out‚ 39.41015625 disk.dm-1.out‚ 272.96875 disk.drbd2080.out‚0 disk.rbd0.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.rbd1.in‚0 disk.rbd0.out‚0# disk.dm-0.out‚28917.44287109375 disk.sda.in‚ 0.01171875 network.sent‚ 2.1801653e+07 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚28917.44287109375 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 272.96875 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0$ memory_percent‚93.53466033935547 disk.drbd1716.in‚0ø8“Î"‚(:ˆ ¸¹æÔ‚ö×ý gpu.0.gpu‚40.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚192.289' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.4578 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ex<ÞhÏ"‚(:Ý ¸¹æÔæ³­€ disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.rbd1.out‚ 39.91796875 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125" proc.memory.rssMB‚ 2621.6953125 disk.sda.out‚ 273.7734375# disk.nvme0n1.out‚28959.48046875 disk.sda3.in‚ 0.01171875$ disk./.usageGB‚302.9120063781738 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0% disk.nvme0c0n1.out‚28959.48046875 disk.drbd2080.out‚0 disk.dm-1.in‚ 0.01171875 disk.sda3.out‚ 273.7734375$ memory_percent‚93.53466033935547* proc.memory.percent‚16.001558303833008 cpu‚25.344606927341587 proc.cpu.threads‚17* disk./.usagePercent‚20.328678908141384& proc.memory.availableMB‚ 1059.28125 network.sent‚ 2.1910674e+07 disk.dm-0.out‚28959.48046875 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 network.recv‚ 2.539785e+06 disk.sda2.in‚0 disk.dm-1.out‚ 273.7734375 disk.drbd1716.out‚0^ØN2Ð"- _wandbruntime‚7153  _runtime‚7153¥AoÇÑ"Á & lr-AdamW/pg1‚2.359442254970417e-05 & lr-AdamW/pg2‚2.359442254970417e-05 trainer/global_step‚8999 % _timestamp‚1.7884520382941852e+09 _runtime‚7153.765797374 _step‚737ñé6ÌÒ"Æ & lr-AdamW/pg1‚2.359442254970417e-05 & lr-AdamW/pg2‚2.359442254970417e-05 trainer/global_step‚8999 % _timestamp‚1.7884520382941852e+09 _runtime‚7153.765797374 _step‚737áf)A2Ó"- _wandbruntime‚7153  _runtime‚7153혼çç'Ô"áO QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 130917044 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 130917042 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚571580468822528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚523436286815744 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7511234940118 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 300834557 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚517228658342102 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚530975295104000 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚571580492762624 trainer/global_step‚8999 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚530974620738560 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚76385964 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚509717423401984 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 169917515 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7908214009856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7538990285312 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 300854597 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _step‚738 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 130917042 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚523436286815744 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7538333922816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚530975295104000 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚517227986864986 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚523436304818688 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚43074374 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944»µª[ì'Õ"æO MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _step‚738 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 130917042 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚523436286815744 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7538333922816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚530975295104000 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚517227986864986 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚523436304818688 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚43074374 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 130917044 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 130917042 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚571580468822528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚523436286815744 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7511234940118 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 300834557 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚517228658342102 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚530975295104000 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚571580492762624 trainer/global_step‚8999 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚530974620738560 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚76385964 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚509717423401984 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 169917515 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 7908214009856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7538990285312 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 300854597âʃʤ2Ö"- _wandbruntime‚7154  _runtime‚7154 y5ýÈ×"ëM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 169956033 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚579551041701376 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_Žhà¦)%batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7539153350656 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7512051375290 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 300887093 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 300867027 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚517283707430510 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚523492657857536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚76394132 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚7154.490768969 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 300867027 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 169935969 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7908922141184 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884520390187411e+09 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7539153350656 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚8999 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 300887093 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚739 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 130931060 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 130931058 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 119473034 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚509772310493184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 130931058 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚509772327532544 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚531032467584000 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚523492639854592 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7539809726464 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚571642143500288 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚517284378907834 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚76394134 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952›`4ö&Ø"ðM SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 169956033 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚579551041701376 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7539153350656 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7512051375290 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 300887093 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 300867027 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚517283707430510 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚523492657857536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚76394132 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚7154.490768969 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 300867027 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 169935969 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7908922141184 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7884520390187411e+09 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7539153350656 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚441 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚8999 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 300887093 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚739 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 130931060 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 130931058 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 119473034 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚509772310493184 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 130931058 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚509772327532544 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚531032467584000 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚523492639854592 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7539809726464 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚571642143500288 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚517284378907834 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚76394134 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680ã//G2Ù"- _wandbruntime‚7154  _runtime‚7154ÉÎ߶YÚ"Ó /loss/FRt1/assignment_loss‚3.670522024998298 2loss/SRqqt3/assignment_loss‚1.3985390242191897 1loss/FBt3/assignment_loss‚0.34279313813402573 /loss/FRt2/detection_loss‚2.0826024491831943 .loss/FBt2/detection_loss‚2.054424544175466 2loss/SRqqt1/assignment_loss‚2.3176474279978057 /loss/FRt3/detection_loss‚1.7296334655644994 /loss/FBt1/detection_loss‚2.4208226568169064 # _timestamp‚1.78845203902047e+09 /loss/FRt3/assignment_loss‚2.140782982110977 0loss/FBt2/assignment_loss‚0.5962557146946583 /loss/FRt4/detection_loss‚2.2491088259015597 /loss/FBt3/detection_loss‚0.9394430642501569 %loss/total_loss‚45.19603676375391 epoch‚7 trainer/global_step‚8999 _runtime‚7154.491988441 0loss/FRt2/assignment_loss‚2.8254513819262668 1loss/SRqqt2/assignment_loss‚1.874712103380347 2loss/SRqqt4/assignment_loss‚1.1667819263921542 1loss/SRqqt4/detection_loss‚1.5287004902281545 _step‚740 0loss/FBt4/assignment_loss‚0.5186580857262015 0loss/SRqqt1/detection_loss‚2.823088518597863 1loss/SRqqt2/detection_loss‚1.7534539318590794 1loss/SRqqt3/detection_loss‚1.4436122024198994 0loss/FRt4/assignment_loss‚3.0653947907335617 .loss/FRt1/detection_loss‚2.880635917186737 1loss/FBt1/assignment_loss‚0.46272162721467996 /loss/FBt4/detection_loss‚2.9102504700422287˜4¦^Û"Ø 1loss/FBt1/assignment_loss‚0.46272162721467996 /loss/FBt4/detection_loss‚2.9102504700422287 /loss/FRt1/assignment_loss‚3.670522024998298 2loss/SRqqt3/assignment_loss‚1.3985390242191897 1loss/FBt3/assignment_loss‚0.34279313813402573 /loss/FRt2/detection_loss‚2.0826024491831943 .loss/FBt2/detection_loss‚2.054424544175466 2loss/SRqqt1/assignment_loss‚2.3176474279978057 /loss/FRt3/detection_loss‚1.7296334655644994 /loss/FBt1/detection_loss‚2.4208226568169064 # _timestamp‚1.78845203902047e+09 /loss/FRt3/assignment_loss‚2.140782982110977 0loss/FBt2/assignment_loss‚0.5962557146946583 /loss/FRt4/detection_loss‚2.2491088259015597 /loss/FBt3/detection_loss‚0.9394430642501569 %loss/total_loss‚45.19603676375391 epoch‚7 trainer/global_step‚8999 _runtime‚7154.491988441 0loss/FRt2/assignment_loss‚2.8254513819262668 1loss/SRqqt2/assignment_loss‚1.874712103380347 2loss/SRqqt4/assignment_loss‚1.1667819263921542 1loss/SRqqt4/detection_loss‚1.5287004902281545 _step‚740 0loss/FBt4/assignment_loss‚0.5186580857262015 0loss/SRqqt1/detection_loss‚2.823088518597863 1loss/SRqqt2/detection_loss‚1.7534539318590794 1loss/SRqqt3/detection_loss‚1.4436122024198994 0loss/FRt4/assignment_loss‚3.0653947907335617 .loss/FRt1/detection_loss‚2.880635917186737ä,0Q¤“Ü"‚(:ˆ ǹæÔû°ý gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.951' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.7902 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0TߤÛbÝ"‚(:× ǹæÔ€ª¦€ disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 40.02734375 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.sda3.out‚ 274.15625 disk.rbd0.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 cpu‚25.34399225642041 proc.cpu.threads‚17# disk.nvme0n1.out‚28998.42578125 disk.drbd2265.out‚0$ memory_percent‚93.53625774383545) proc.memory.availableMB‚ 1059.01953125* proc.memory.percent‚16.001558303833008 network.recv‚ 2.54298e+06 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.drbd2266.out‚0$ disk./.usageGB‚302.8677520751953 disk.dm-0.out‚28998.42578125 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875) disk./.usagePercent‚20.32570896473666 disk.sda1.out‚0 disk.dm-1.out‚ 274.15625 disk.drbd2080.out‚0 disk.rbd0.out‚0 network.sent‚ 2.1915923e+07 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.sda.out‚ 274.15625 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚28998.425781250‘l›“Þ"‚(:ˆ Ö¹æÔÙÜ”ý gpu.0.gpu‚39.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.047' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8094 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¢ÔŽ+pß"‚(:å Ö¹æÔŽ‡ÿ disk.sda3.out‚ 274.765625 disk.drbd2266.in‚0 network.sent‚ 2.2025404e+07 disk.rbd0.in‚0 disk.sda.out‚ 274.765625 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17 disk.rbd0.out‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚29039.79345703125 disk.dm-1.in‚ 0.01171875 disk.drbd2266.out‚0* disk./.usagePercent‚20.327415254952374 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚29039.79345703125( disk.nvme0c0n1.out‚29039.79345703125$ memory_percent‚93.53547096252441* proc.memory.percent‚16.001558303833008 cpu‚25.343388253331725 network.recv‚ 2.550222e+06$ disk./.usageGB‚302.8931770324707 disk.sda1.in‚0 disk.rbd1.in‚0 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 40.08984375 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda1.out‚0 disk.dm-1.out‚ 274.765625 disk.drbd2080.out‚0( proc.memory.availableMB‚ 1059.1484375 disk.sda2.in‚0Ìø(r“à"‚(:ˆ å¹æÔ×ó‹ý gpu.0.gpu‚38.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.917' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.5834 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0N¼P`á"‚(:Õ å¹æÔïÈ™ÿ network.sent‚ 2.2034137e+07 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚29076.50048828125 disk.rbd1.in‚0$ memory_percent‚93.53713989257812$ disk./.usageGB‚302.8991508483887 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd1.out‚ 40.08984375 disk.dm-1.out‚275.25" proc.memory.rssMB‚ 2621.6953125 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0& disk.nvme0n1.out‚29076.50048828125 disk.sda3.out‚275.25 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd1716.out‚0 cpu‚25.342777761609955 network.recv‚ 2.560324e+06 disk.sda.out‚275.25 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0 disk.sda2.out‚0* disk./.usagePercent‚20.327816162750995 disk.drbd2266.in‚0# disk.dm-0.out‚29076.50048828125* proc.memory.percent‚16.001558303833008 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.drbd1716.in‚0$ proc.memory.availableMB‚1058.875Ò1ÖF2â"- _wandbruntime‚7191  _runtime‚7191D½ÍÈã" % _timestamp‚1.7884520764617262e+09 _runtime‚ 7191.93333988 _step‚741 ' lr-AdamW/pg1‚2.2886610847684678e-05 ' lr-AdamW/pg2‚2.2886610847684678e-05 trainer/global_step‚9049/¾áýÍä"Ç _runtime‚ 7191.93333988 _step‚741 ' lr-AdamW/pg1‚2.2886610847684678e-05 ' lr-AdamW/pg2‚2.2886610847684678e-05 trainer/global_step‚9049 % _timestamp‚1.7884520764617262e+09å)ð¼\2å"- _wandbruntime‚7191  _runtime‚7191¨Ð•«¤æ"àO TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 302481449 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 131617842 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7951088420352 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7579319728640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7552070279526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7579976091136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚76794364 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚526253938758144 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚520014028732394 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚582614094126080 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚526253956761088 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 trainer/global_step‚9049 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 170843567 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚533833258486784 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚444 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚533833258486784 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 120095852 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚76794366 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚43301486 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _runtime‚7191.934321903 _step‚742 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 131617844 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 120095406 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚574663005705728 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7579976091136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚526253956761088 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 170863605 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872û…¡Bë'ç"åO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚582614094126080 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚526253956761088 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 337538560 trainer/global_step‚9049 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 170843567 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚533833258486784 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚444 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚533833258486784 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 120095852 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚76794366 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚43301486 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _runtime‚7191.934321903 _step‚742 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 131617844 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 120095406 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚574663005705728 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7579976091136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚526253956761088 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 170863605 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 302481449 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 131617842 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7951088420352 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7579319728640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7552070279526 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7579976091136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚76794364 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚526253938758144 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚520014028732394 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚76802534 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 302513945 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 _step‚743 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 302513945 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚533890430953472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚526310309799936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7580139156480 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚574724656443392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 _runtime‚7192.640443845 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 $ _timestamp‚1.788452077168386e+09 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 120108225 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚43306145 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7952107804672 trainer/global_step‚9049 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚533891105332224 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7580795532288 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚43305693 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚76802532 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚582676788188160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚574724680383488 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚582676596443136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚512517534060544 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 302493879 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚526310291796992 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 302493879 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 170882085 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 170862021 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7580795532288 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 170882085 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 131631858 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚520070420775242 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚520069749297918 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 131631858 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚512517517021184 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7951939999744 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944×Br1Žê"ïM E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7951939999744 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 337538560 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7552886714698 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7552232276734 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 170862021 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚526310309799936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚76802534 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 302513945 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 _step‚743 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 302513945 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚533890430953472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚526310309799936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7580139156480 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚574724656443392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 _runtime‚7192.640443845 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 $ _timestamp‚1.788452077168386e+09 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 120108225 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚43306145 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 7952107804672 trainer/global_step‚9049 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚533891105332224 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7580795532288 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚43305693 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚76802532 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚582676788188160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚574724680383488 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 995§ÖLg 5586048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚582676596443136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚512517534060544 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 302493879 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚526310291796992 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 302493879 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 170882085 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 170862021 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7580795532288 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 170882085 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 131631858 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚520070420775242 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚520069749297918 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 131631858 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚512517517021184 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944ç¯oÉ^2ë"- _wandbruntime‚7192  _runtime‚7192ÅζZì"Ô /loss/FRt2/detection_loss‚2.2409941414181067 trainer/global_step‚9049 % _timestamp‚1.7884520771699958e+09 /loss/FBt2/detection_loss‚1.8377952476342518 .loss/FRt1/detection_loss‚3.404137873649597 /loss/FBt2/assignment_loss‚0.451354617278816 1loss/FBt4/assignment_loss‚0.38017268758265743 1loss/SRqqt3/detection_loss‚1.1802094683927649 /loss/FRt1/assignment_loss‚4.507476830482483 1loss/FBt3/assignment_loss‚0.21233228312696484 /loss/FRt3/detection_loss‚1.8381513446951523 /loss/FBt3/detection_loss‚0.8280713206984931 %loss/total_loss‚44.87808059165154 2loss/SRqqt4/assignment_loss‚1.1758610588114928 0loss/SRqqt1/detection_loss‚2.615362850683076 0loss/FRt2/assignment_loss‚3.4039819909601796 2loss/SRqqt1/assignment_loss‚2.2925009557179044 /loss/FBt1/detection_loss‚2.8543884328433444 -loss/FBt4/detection_loss‚2.76500191539526 0loss/FRt3/assignment_loss‚1.9814268289468226 0loss/FRt4/assignment_loss‚2.6075320839881897 1loss/SRqqt3/assignment_loss‚1.032160182807631 0loss/SRqqt4/detection_loss‚1.517069582334336 /loss/FRt4/detection_loss‚2.0013967061584648 epoch‚7 _step‚744 2loss/SRqqt2/assignment_loss‚1.4227888644033788 1loss/FBt1/assignment_loss‚0.49905317952953965 1loss/SRqqt2/detection_loss‚1.8288601441126244 _runtime‚7192.641514967*6º&_í"Ù 1loss/FBt1/assignment_loss‚0.49905317952953965 1loss/SRqqt2/detection_loss‚1.8288601441126244 _runtime‚7192.641514967 /loss/FRt2/detection_loss‚2.2409941414181067 trainer/global_step‚9049 % _timestamp‚1.7884520771699958e+09 /loss/FBt2/detection_loss‚1.8377952476342518 .loss/FRt1/detection_loss‚3.404137873649597 /loss/FBt2/assignment_loss‚0.451354617278816 1loss/FBt4/assignment_loss‚0.38017268758265743 1loss/SRqqt3/detection_loss‚1.1802094683927649 /loss/FRt1/assignment_loss‚4.507476830482483 1loss/FBt3/assignment_loss‚0.21233228312696484 /loss/FRt3/detection_loss‚1.8381513446951523 /loss/FBt3/detection_loss‚0.8280713206984931 %loss/total_loss‚44.87808059165154 2loss/SRqqt4/assignment_loss‚1.1758610588114928 0loss/SRqqt1/detection_loss‚2.615362850683076 0loss/FRt2/assignment_loss‚3.4039819909601796 2loss/SRqqt1/assignment_loss‚2.2925009557179044 /loss/FBt1/detection_loss‚2.8543884328433444 -loss/FBt4/detection_loss‚2.76500191539526 0loss/FRt3/assignment_loss‚1.9814268289468226 0loss/FRt4/assignment_loss‚2.6075320839881897 1loss/SRqqt3/assignment_loss‚1.032160182807631 0loss/SRqqt4/detection_loss‚1.517069582334336 /loss/FRt4/detection_loss‚2.0013967061584648 epoch‚7 _step‚744 2loss/SRqqt2/assignment_loss‚1.4227888644033788èzM“î"‚(:ˆ ô¹æÔïÅþü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.949' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5898 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÇóØÌgï"‚(:Ü ô¹æÔÍ’ÿ disk.drbd2265.out‚0 disk.drbd1716.out‚0) disk./.usagePercent‚20.32841240429824 disk.rbd0.out‚0 disk.sda1.out‚0 disk.dm-1.out‚ 275.8984375$ disk.nvme0c0n1.out‚ 29122.3046875 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 275.8984375 disk.rbd1.in‚0" disk.nvme0n1.out‚ 29122.3046875 disk.drbd2080.out‚0$ memory_percent‚93.53890419006348( proc.memory.availableMB‚ 1058.5859375" proc.memory.rssMB‚ 2621.6953125 network.recv‚ 2.56546e+06 disk.sda1.in‚0 disk.sda2.in‚0* proc.memory.percent‚16.001558303833008 cpu‚25.342170622285405 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.dm-0.out‚ 29122.3046875 disk.sda.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2265.in‚0$ disk./.usageGB‚302.9080352783203 disk.sda3.out‚ 275.8984375 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125 proc.cpu.threads‚17 network.sent‚ 2.2040607e+07 disk.rbd1.out‚ 40.1220703125?3(žð"‚(:“ ƒºæÔŠÍÔý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚179.366' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.873200000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0MÖcxñ"‚(:í ƒºæÔüï¥ÿ disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.rbd0.out‚0# disk.dm-0.out‚29147.16064453125 disk.sda3.out‚ 276.43359375 disk.sda.out‚ 276.43359375* proc.memory.percent‚16.001558303833008( disk.nvme0c0n1.out‚29147.16064453125 disk.dm-0.in‚0.078125& disk.nvme0n1.out‚29147.16064453125 disk.drbd2265.out‚0$ memory_percent‚93.53156089782715 cpu‚25.341564518682524 network.recv‚ 2.570915e+06 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 40.3017578125 disk.drbd1716.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda1.out‚0 proc.cpu.threads‚17 network.sent‚ 2.2148274e+07 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875( proc.memory.availableMB‚ 1059.7890625* disk./.usagePercent‚20.328811008029053$ disk./.usageGB‚302.9139747619629 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.dm-1.out‚ 276.43359375 disk.drbd1716.out‚0 disk.drbd2265.in‚0hòOË2ò"- _wandbruntime‚7229  _runtime‚7229/¤€¸Éó"à ' lr-AdamW/pg2‚2.2187230164512357e-05 trainer/global_step‚9099 % _timestamp‚1.7884521139965081e+09 _runtime‚7229.468105819 _step‚745 ' lr-AdamW/pg1‚2.2187230164512357e-05žéDÎô"È ' lr-AdamW/pg1‚2.2187230164512357e-05 ' lr-AdamW/pg2‚2.2187230164512357e-05 trainer/global_step‚9099 % _timestamp‚1.7884521139965081e+09 _runtime‚7229.468105819 _step‚745é“3%Ñ2õ"- _wandbruntime‚7229  _runtime‚7229a¼»¡ç'ö"áO E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚585739929900544 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7620961896960 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 304088261 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚529071608703488 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚522800742076918 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚536692570600448 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 171789657 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 132318644 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 171789657 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7620305534464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚746 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚9099 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 132318644 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚577745566529024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 304108301 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 120732826 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _runtime‚7229.469124241 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 132318642 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚77202764 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7884521139972126e+09 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 171769619 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚536691896235008 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7994193456128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7620961896960 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚473 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7592905618934 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _runtime‚7229.469124241 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 132318642 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚77202764 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7884521139972126e+09 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 171769619 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚536691896235008 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 7994193456128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7620961896960 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚473 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7592905618934 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7621781338112 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 304140797 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 120746002 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 304120731 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚536749743080448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 120745528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 304140797 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚77210932 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚529127943739392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7621781338112 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚577807193326592 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚515262723549184 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 132332660 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7593722054106 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚522855791165326 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚515262740588544 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 132332660 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 171808137 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚474 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7995055156736 _runtime‚7230.204639706 _step‚747 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 171788073 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚529127943739392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚529127961742336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7593067616142 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7621124962304 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 132332658 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 304140797 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 120746002 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 304120731 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚536749743080448 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 120745528 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 304140797 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚77210932 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚529127943739392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7621781338112 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚577807193326592 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚515262723549184 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 132332660 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7593722054106 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚522855791165326 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚515262740588544 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 132332660 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 171808137 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚474 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 7995055156736 _runtime‚7230.204639706 _step‚747 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 171788073 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚529127943739392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚529127961742336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7593067616142 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7621124962304 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 132332658 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7621781338112 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8037454992896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚531889242642944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚43757779 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚77611164 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 172715709 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 172695671 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚43758223 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚539550533983232 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 172695671 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8037295563264 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚525586112467210 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 133019444 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7884521521486573e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 305715113 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 133019442 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚539551208348672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚531889260645888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚77611166 D:DeviceStatsMons$sZitor.on_train_batch_start/active.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚531889242642944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚525586783944326 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7633086520586 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚539551208348672 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7633740958342 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚9149 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚531889260645888 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7661291340288 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚588865558405120 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414½¡Àì'‰#æO ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚588865558405120 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 305735153 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8037454992896 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚531889242642944 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚43757779 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚77611164 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 172715709 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 172695671 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚43758223 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚539550533983232 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 172695671 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8037295563264 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚525586112467210 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 133019444 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 % _timestamp‚1.7884521521486573e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 305715113 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 133019442 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚539551208348672 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚531889260645888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚77611166 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚531889242642944 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚525586783944326 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7633086520586 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚539551208348672 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7633740958342 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚9149 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚531889260645888 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7661291340288î4 1N2Š#- _wandbruntime‚7268  _runtime‚7268ƒlÚñ&‹#ëM I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚531945595681792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 121381691 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 133033460 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7662767143936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚77619332 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚588928075901952 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 133033460 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚580889754149888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚518007930077184 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚539608380828672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8038164432896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7662110768128 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 133033458 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚531945595681792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 121382122 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 305747583 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚43762359 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7634557393514 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 172734189 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 133033458 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 172714125 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 _runtime‚7268.345147308 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 305767649 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 172734189 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 305767649 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 trainer/global_step‚9149 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚539608380828672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7633902955550 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚525641833032734 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚531945613684736 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚588927894642688 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚751 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚4309 >®ö&Œ#ðM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8038164432896 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7662110768128 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 133033458 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚531945595681792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 121382122 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 305747583 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚43762359 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7634557393514 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 172734189 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 133033458 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 172714125 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 _runtime‚7268.345147308 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 305767649 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 172734189 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 305767649 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 trainer/global_step‚9149 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚539608380828672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7633902955550 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚525641833032734 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚531945613684736 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚588927894642688 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚751 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚531945595681792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 121381691 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 133033460 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7662767143936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚429 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚77619332 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚588928075901952 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 133033460 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚580889754149888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚518007930077184 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚539608380828672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088ïÏa[T2#- _wandbruntime‚7268  _runtime‚7268E'k—–Ž#Ò 2loss/SRqqt2/assignment_loss‚1.2677936740219593 _runtime‚7268.346832939 0loss/FBt3/assignment_loss‚0.2232076565041136 -loss/FRt4/detection_loss‚2.54636340874892 epoch‚7 1loss/FBt1/assignment_loss‚0.23995635315165892 0loss/FBt2/assignment_loss‚0.2970420118759979 %loss/total_loss‚46.12275789770618 % _timestamp‚1.7884521528753407e+09 0loss/FRt2/assignment_loss‚¥j/¿Â2.6082742226620517 0loss/FRt4/assignment_loss‚2.6645016425695176 2loss/SRqqt3/assignment_loss‚1.1553494008128635 0loss/FBt4/assignment_loss‚1.3296310529112816 1loss/SRqqt1/detection_loss‚2.7061592888832093 1loss/SRqqt4/detection_loss‚1.5882042763133843 _step‚752 1loss/SRqqt1/assignment_loss‚2.722223402261734 /loss/FBt3/detection_loss‚0.6932297310830183 trainer/global_step‚9149 .loss/FRt2/detection_loss‚2.168517127633095 /loss/FRt1/assignment_loss‚4.467637687921524 .loss/FRt1/detection_loss‚3.255439738432566 1loss/SRqqt3/detection_loss‚1.2028332509410877 .loss/FBt1/detection_loss‚3.047965028456279 0loss/FRt3/assignment_loss‚1.9724249223868051 /loss/FRt3/detection_loss‚1.4673933578034242 /loss/FBt2/detection_loss‚2.6595505638556047 /loss/FBt4/detection_loss‚2.7916122674942017 2loss/SRqqt4/assignment_loss‚1.3811963938642293 0loss/SRqqt2/detection_loss‚1.666251437117656~ž½²]#× .loss/FRt2/detection_loss‚2.168517127633095 /loss/FRt1/assignment_loss‚4.467637687921524 .loss/FRt1/detection_loss‚3.255439738432566 1loss/SRqqt3/detection_loss‚1.2028332509410877 .loss/FBt1/detection_loss‚3.047965028456279 0loss/FRt3/assignment_loss‚1.9724249223868051 /loss/FRt3/detection_loss‚1.4673933578034242 /loss/FBt2/detection_loss‚2.6595505638556047 /loss/FBt4/detection_loss‚2.7916122674942017 2loss/SRqqt4/assignment_loss‚1.3811963938642293 0loss/SRqqt2/detection_loss‚1.666251437117656 2loss/SRqqt2/assignment_loss‚1.2677936740219593 _runtime‚7268.346832939 0loss/FBt3/assignment_loss‚0.2232076565041136 -loss/FRt4/detection_loss‚2.54636340874892 epoch‚7 1loss/FBt1/assignment_loss‚0.23995635315165892 0loss/FBt2/assignment_loss‚0.2970420118759979 %loss/total_loss‚46.12275789770618 % _timestamp‚1.7884521528753407e+09 0loss/FRt2/assignment_loss‚2.6082742226620517 0loss/FRt4/assignment_loss‚2.6645016425695176 2loss/SRqqt3/assignment_loss‚1.1553494008128635 0loss/FBt4/assignment_loss‚1.3296310529112816 1loss/SRqqt1/detection_loss‚2.7061592888832093 1loss/SRqqt4/detection_loss‚1.5882042763133843 _step‚752 1loss/SRqqt1/assignment_loss‚2.722223402261734 /loss/FBt3/detection_loss‚0.6932297310830183 trainer/global_step‚9149ð›û8“#‚(:ˆ ¿ºæÔ½¹Œý gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.627' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9254 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0†­Wm‘#‚(:â ¿ºæÔ„¦™" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0* proc.memory.percent‚16.001558303833008 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 network.sent‚ 2.2358991e+07% disk./.usageGB‚302.92364501953125 disk.sda.in‚ 0.01171875 disk.sda.out‚ 278.9140625 disk.rbd1.out‚ 40.8486328125 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.sda1.in‚0 disk.rbd1.in‚0 cpu‚25.339190019088257 disk.dm-1.out‚ 278.9140625 disk.drbd2080.out‚0 disk.dm-0.out‚29312.37890625 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125) proc.memory.availableMB‚ 1059.69140625 network.recv‚ 2.598494e+06 disk.drbd2265.in‚0 disk.sda2.in‚0* disk./.usagePercent‚20.329459987128352 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.sda3.out‚ 278.9140625 disk.drbd2266.out‚0# memory_percent‚93.5321569442749 disk.drbd2265.out‚0% disk.nvme0c0n1.out‚29312.37890625 disk.sda2.out‚0# disk.nvme0n1.out‚29312.37890625Ùƒ§¡ž’#‚(:“ κæÔ·±÷ü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.551' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.910199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ç,Ui“#‚(:Þ κæÔý¤ÿ disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚29352.4990234375 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.drbd2080.out‚0 disk.sda2.in‚0 disk.rbd1.in‚0$ memory_percent‚93.53213310241699* proc.memory.percent‚16.001558303833008 network.recv‚ 2.606341e+06 disk.sda1.in‚0 disk.sda3.in‚ 0.01171875% disk.nvme0n1.out‚29352.4990234375 disk.sda.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.rbd0.out‚0 proc.cpu.threads‚17 disk.sda1.out‚0 disk.dm-0.in‚0.078125 cpu‚25.338578121013047 disk.sda3.out‚279.5625* disk./.usagePercent‚20.330888253161877 disk.sda2.out‚0 disk.rbd1.out‚ 40.9970703125 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚279.5625 disk.drbd2265.out‚0" disk.dm-0.out‚29352.4990234375 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2266.out‚0( proc.memory.availableMB‚ 1059.6953125" proc.memory.rssMB‚ 2621.6953125$ disk./.usageGB‚302.9449272155762 disk.sda.out‚279.5625 disk.drbd2080.in‚0 network.sent‚ 2.2468507e+07ÜöÝ“”#‚(:ˆ ݺæÔ•Îþü gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.724' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.9448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0LýÙçr•#‚(:ç ݺæÔ¬…‚€ network.recv‚ 2.609649e+06 disk.nvme0c0n1.in‚0.078125" disk.dm-0.out‚29390.7333984375 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 cpu‚25.337981577713336 network.sent‚ 2.2473689e+07* disk./.usagePercent‚20.331584337647477 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.rbd1.in‚0 disk.rbd1.out‚ 40.9970703125% disk.nvme0n1.out‚29390.7333984375 disk.drbd1716.out‚0 disk.nvme0n1.in‚0.078125$ memory_percent‚93.53365898132324( proc.memory.availableMB‚ 1059.4453125 disk.rbd0.out‚0 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 280.0234375 disk.dm-0.in‚0.078125 disk.sda.out‚ 280.0234375 disk.sda2.out‚0* proc.memory.percent‚16.001558303833008 disk.rbd0.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚302.9552993774414 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚29390.7333984375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.dm-1.out‚ 280.0234375 disk.drbd2265.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.drbd1716.in‚0" proc.memory.rssMB‚ 2621.6953125q÷Y2–#- _wandbruntime‚7305  _runtime‚7305$AóUÇ—#Á trainer/global_step‚9199 # _timestamp‚1.78845218976764e+09 _runtime‚7305.239260167 _step‚753 ' lr-AdamW/pg1‚2.0814343066867975e-05 ' lr-AdamW/pg2‚2.0814343066867975e-05uê‚̘#Æ ' lr-AdamW/pg2‚2.0814343066867975e-05 trainer/global_step‚9199 # _timestamp‚1.78845218976764e+09 _runtime‚7305.239260167 _step‚753 ' lr-AdamW/pg1‚2.0814343066867975e-05ñšÝzâ2™#- _wandbruntime‚7305  _runtime‚7305˜ç'š#áO GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7702933508608 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚78019566 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚9199 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 307362005 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚520698249513984 _step‚754 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚438 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚43986351 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7702277146112 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 122006355 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 173621723 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚542409171731456 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚591990918480896 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚534706912588288 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚9199 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 307362005 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚520698249513984 _step‚754 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚438 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚43986351 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7702277146112 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 122006355 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 173621723 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚542409171731456 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚591990918480896 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚534706912588288 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚520753136605184 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 307394501 trainer/global_step‚9199 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚528428546377466 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7675392732922 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7703096573952 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7703752949760 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚78027732 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚542467018576896 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚7305.937779229 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚583972291033088 $ _timestamp‚1.788452190465746e+09 _step‚755 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 307374435 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚534763247624192 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8081164872704 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7703096573952 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 122019017 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 173660241 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚534763265627136 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 173640177 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚78027734 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8081326386176 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 173640177 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚43991283 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚534763247624192 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚542467018576896 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 122018578 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚592053617419264 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 173660241 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 133734260 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 133734258 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 307394501 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2ß„L›õ&ž#ïM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚534763247624192 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚542467018576896 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 122018578 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚592053617419264 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 173660241 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 133734260 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 133734258 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 307394501 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚520753136605184 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 307394501 trainer/global_step‚9199 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚528428546377466 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7675392732922 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7703096573952 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7703752949760 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚78027732 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚542467018576896 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚7305.937779229 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚583972291033088 $ _timestamp‚1.788452190465746e+09 _step‚755 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 307374435 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚534763247624192 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8081164872704 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7703096573952 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 122019017 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 173660241 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚534763265627136 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 173640177 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚78027734 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8081326386176 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 173640177 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚43991283ór3Hw2Ÿ#- _wandbruntime‚7305  _runtime‚7305ƒ)cZ #Ô 1loss/SRqqt4/detection_loss‚1.6296983055489243 &loss/total_loss‚44.172415902397915 /loss/FRt1/assignment_loss‚4.183055410782496 2loss/SRqqt1/assignment_loss‚2.1822750904867725 1loss/FBt3/assignment_loss‚0.20666845613051046 -loss/FRt2/detection_loss‚2.13583720665352 2loss/SRqqt2/assignment_loss‚1.2433728826128774 _runtime‚7305.938882682 1loss/SRqqt4/assignment_loss‚1.034292070651954 /loss/FRt1/detection_loss‚3.0131308200458684 1loss/SRqqt3/detection_loss‚1.2276485505281016 /loss/FBt1/detection_loss‚2.4981973727544147 epoch‚7 trainer/global_step‚9199 1loss/FBt2/assignment_loss‚0.28302753104447265 /loss/FRt4/detection_loss‚2.0944034011604695 /loss/SRqqt1/detection_loss‚2.58682749156029 /loss/FRt2/assignment_loss‚2.897285400652418 0loss/FRt3/assignment_loss‚1.6371541481751661 /loss/FRt4/assignment_loss‚3.211503865627142 1loss/FBt1/assignment_loss‚0.46394174741840666 /loss/FRt3/detection_loss‚2.2146650531735177 1loss/SRqqt2/detection_loss‚1.7542759458775874 .loss/FBt2/detection_loss‚2.415211377774968 /loss/FBt3/detection_loss‚0.8306525237599595 /loss/FBt4/detection_loss‚2.6869815985361734 _step‚756 2loss/SRqqt3/assignment_loss‚1.1167159163305769 0loss/FBt4/assignment_loss‚0.6255937351113263 % _timestamp‚1.7884521904673665e+09[‹—_¡#Ù 1loss/SRqqt4/assignment_loss‚1.034292070651954 /loss/FRt1/detection_loss‚3.0131308200458684 1loss/SRqqt3/detection_loss‚1.2276485505281016 /loss/FBt1/detection_loss‚2.4981973727544147 epoch‚7 trainer/global_step‚9199 1loss/FBt2/assignment_loss‚0.28302753104447265 /loss/FRt4/detection_loss‚2.0944034011604695 /loss/SRqqt1/detection_loss‚2.58682749156029 /loss/FRt2/assignment_loss‚2.897285400652418 0loss/FRt3/assignment_loss‚1.6371541481751661 /loss/FRt4/assignment_loss‚3.211503865627142 1loss/FBt1/assignment_loss‚0.46394174741840666 /loss/FRt3/detection_loss‚2.2146650531735177 1loss/SRqqt2/detection_loss‚1.7542759458775874 .loss/FBt2/detection_loss‚2.415211377774968 /loss/FBt3/detection_loss‚0.8306525237599595 /loss/FBt4/detection_loss‚2.6869815985361734 _step‚756 2loss/SRqqt3/assignment_loss‚1.1167159163305769 0loss/FBt4/assignment_loss‚0.6255937351113263 % _timestamp‚1.7884521904673665e+09 1loss/SRqqt4/detection_loss‚1.6296983055489243 &loss/total_loss‚44.172415902397915 /loss/FRt1/assignment_loss‚4.183055410782496 2loss/SRqqt1/assignment_loss‚2.1822750904867725 1loss/FBt3/assignment_loss‚0.20666845613051046 -loss/FRt2/detection_loss‚2.13583720665352 2loss/SRqqt2/assignment_loss‚1.2433728826128774 _runtime‚7305.938882682ô\m•c¢#‚(:’ ìºæÔºí›ý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.33' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.666000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ï K'n£#‚(:ã ìºæÔÔÒÁÿ disk.drbd2266.out‚0 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17% disk./.usageGB‚302.91418838500977 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 280.66796875$ memory_percent‚93.53387355804443 disk.rbd0.out‚0 disk.dm-0.out‚29431.15234375 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0) proc.memory.availableMB‚ 1059.41015625 disk.sda1.out‚0 disk.dm-1.out‚ 280.66796875 disk.nvme0c0n1.in‚0.078125 network.sent‚ 2.247995e+07 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 280.66796875 disk.sda2.out‚0 disk.rbd1.out‚ 41.1455078125% disk.nvme0c0n1.out‚29431.15234375 cpu‚25.337397311594632) disk./.usagePercent‚20.32882534445097 disk.dm-1.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 network.recv‚ 2.61424e+06 disk.rbd1.in‚0# disk.nvme0n1.out‚29431.15234375* proc.memory.percent‚16.001558303833008(ˆž¤#‚(:“ ûºæÔ¿Œ™ý gpu.0.gpu‚43.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚145.402' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.080399999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0àÑ#ôj¥#‚(:ß ûºæԧ̰ÿ disk.rbd0.in‚0 disk.rbd0.out‚0 disk.rbd1.in‚0 disk.nvme0c0n1.in‚0.078125$ memory_percent‚93.53234767913818 network.sent‚ 2.2587777e+07 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 281.2734375 cpu‚25.336812888663584 proc.cpu.threads‚17 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 41.2392578125 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0) proc.memory.availableMB‚ 1059.66015625 disk.dm-0.out‚ 29469.6953125 disk.sda1.in‚0 disk.sda1.out‚0 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2621.6953125$ disk.nvme0c0n1.out‚ 29469.6953125* disk./.usagePercent‚20.330796602464616 disk.nvme0n1.in‚0.078125" disk.nvme0n1.out‚ 29469.6953125 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.dm-1.out‚ 281.2734375 disk.drbd2265.out‚0* proc.memory.percent‚16.001558303833008 disk.sda.out‚ 281.2734375 network.recv‚ 2.619871e+06$ disk./.usageGB‚302.9435615539551æê“02¦#- _wandbruntime‚7343  _runtime‚7343ìÕ=Oȧ# _step‚757 ' lr-AdamW/pg1‚2.0141122760009314e-05 ' lr-AdamW/pg2‚2.0141122760009314e-05 trainer/global_step‚9249 $ _timestamp‚1.788452227755872e+09 _runtime‚7343.227499782†Xͨ#Ç _runtime‚7343.227499782 _step‚757 ' lr-AdamW/pg1‚2.0141122760009314e-05 ' lr-AdamW/pg2‚2.0141122760009314e-05 trainer/global_step‚9249 $ _timestamp‚1.788452227755872e+09õF‹2©#- _wandbruntime‚7343  _runtime‚7343ý±¡¥ç'ª#áO JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚545267809479680 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚595116445333504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8123447330304 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚7343.228485502 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 134421044 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚531158867679142 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚9249 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 134421044 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚78427964 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚445 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚523443439002624 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 122642974 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚545267809479680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚78427966 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚44215455 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚586993153238528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚545268483845120 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 122643421 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚523443456041984 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 308968817 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚44215010 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚595116624508928 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚447 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235328 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7714757199402 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7715411637158 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 174567813 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚537524564530688 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚537524564530688 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚586993177178624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚537524546527744 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7743919314432 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 174567813 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 % _timestamp‚1.7884522277566233e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 174547775 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚537524546527744 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 308988857 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8123292094976 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7743919314432 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚545268483845120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7743262951936 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 174547775 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚758 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7743262951936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 134421042 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚531158196202026 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430”Ì]=«#æO BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚545267809479680 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚595116445333504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8123447330304 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚7343.228485502 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 134421044 YEDeviceStatsMonitor.on_train_batch_start/requested_by꺢5¯$tes.all.allocated‚531158867679142 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚9249 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 134421044 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚78427964 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚445 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚523443439002624 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 122642974 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚545267809479680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚78427966 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚44215455 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚586993153238528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚545268483845120 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 122643421 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚523443456041984 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 308968817 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚44215010 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚595116624508928 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚447 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235328 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7714757199402 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7715411637158 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 174567813 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚537524564530688 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚537524564530688 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚586993177178624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚537524546527744 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7743919314432 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 174567813 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 % _timestamp‚1.7884522277566233e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 174547775 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚537524546527744 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 308988857 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8123292094976 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7743919314432 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚545268483845120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7743262951936 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 174547775 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _step‚758 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7743262951936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 134421042 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚531158196202026 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1öŸßµT2¬#- _wandbruntime‚7343  _runtime‚7343É‘bñ&­#ëM S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚531213916767550 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚9249 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 134435060 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 309021353 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 174586293 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _step‚759 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7716228072330 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8124312468480 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚7343.931787981 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 309001287 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 174586293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7744082379776 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚537580917569536 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 309001287 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 309021353 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7744082379776 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚44219536 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚44219984 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚595179140384768 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚523498343133184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 122655668 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚587054827916288 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚537580917569536 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚531214588244874 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 134435058 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚78436132 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚595178950736896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 174566229 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7715573634366 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 134435058 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚537580899566592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚523498360172544 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 122656118 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 % _timestamp‚1.7884522284596171e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚545324981946368 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚587054803976192|ÁE÷ö&®#ðM L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 122655668 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚587054827916288 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚537580917569536 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚531214588244874 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 134435058 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚78436132 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚595178950736896 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 174566229 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7715573634366 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 134435058 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚537580899566592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚523498360172544 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 122656118 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 % _timestamp‚1.7884522284596171e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚545324981946368 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚587054803976192 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚531213916767550 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚9249 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 134435060 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 309021353 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 174586293 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 _step‚759 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7716228072330 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8124312468480 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚7343.931787981 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 309001287 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 174586293 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7744082379776 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚537580917569536 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 309001287 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 309021353 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7744082379776 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚44219536 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚44219984 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚595179140384768 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚523498343133184 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚548126447227904 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚548127121593344 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8166378928128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 123279801 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚78836364 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8166213207040 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7784248757760 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 $ _timestamp‚1.788452265289743e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 175473827 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚526188645530624 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚540342198470144 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚590075690121728 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 123279345 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚533944238069434 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7756246976566 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7784248757760 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚590075714061824 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 trainer/global_step‚9299 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 135121842 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚44443435 _runtime‚ 7380.76159992 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 175493865 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7784905120256 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 _step‚762 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚78836366 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚526188662569984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 310595669 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚540342198470144 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7784905120256 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0~xË!ê'½#äO QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 175473827 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚526188645530624 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚540342198470144 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚590075690121728 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 123279345 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚533944238069434 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7756246976566 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7784248757760 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚590075714061824 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 trainer/global_step‚9299 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 135121842 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚44443435 _runtime‚ 7380.76159992 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 175493865 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7784905120256 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 _step‚762 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚78836366 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚526188662569984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 310595669 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚540342198470144 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7784905120256 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 310615709 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚548126447227904 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚548127121593344 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8166378928128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 123279801 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚78836364 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8166213207040 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7784248757760 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 $ _timestamp‚1.788452265289743e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605ú䵶2¾#- _wandbruntime‚7381  _runtime‚7381nt:¨¿#ëM L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 123292090 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7785724561408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚540398569511936 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8167235886080 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7756408973774 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 310628139 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 175492281 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚590137364799488 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚540398551508992 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚534000630112282 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 175512345 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚590137340859392 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚598304600685568 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚548184294073344 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 310648205 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚526243566700544 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7785068185600 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 67437Ó÷_I 8752 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚78844534 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚44447991 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚598304415232000 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚533999958634958 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚44447558 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚548184294073344 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8167074372608¸›Ãö&À#ðM L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 310648205 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚526243566700544 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7785068185600 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚78844534 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚44447991 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚598304415232000 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚533999958634958 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚44447558 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚548184294073344 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8167074372608 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 123292090 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7785724561408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚540398569511936 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8167235886080 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7756408973774 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 310628139 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 175492281 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚590137364799488 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚540398551508992 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚534000630112282 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 175512345 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚590137340859392 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚598304600685568 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚548184294073344 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0ûð»åp2Á#- _wandbruntime‚7381  _runtime‚7381°ÍÓ¢WÂ#Ñ .loss/FRt3/assignment_loss‚2.05558074655987 trainer/global_step‚9299 0loss/FBt1/assignment_loss‚0.5792445898469951 1loss/FBt4/assignment_loss‚0.18104832557340464 .loss/FRt4/detection_loss‚2.280393601981578 1loss/SRqqt1/detection_loss‚2.4027593380365615 /loss/FBt2/detection_loss‚2.1110439234309726 _runtime‚7381.467419139 _step‚764 0loss/FRt2/assignment_loss‚2.9050712744394938 2loss/SRqqt2/assignment_loss‚1.2913358476944268 /loss/FBt2/assignment_loss‚0.719354274786181 /loss/FBt3/assignment_loss‚0.255948271968622 /loss/FBt1/detection_loss‚2.5219691263304815 /loss/FBt3/detection_loss‚0.6579934500964979 $ _timestamp‚1.788452265995899e+09 2loss/SRqqt1/assignment_loss‚1.9243775182045424 2loss/SRqqt4/assignment_loss‚1.4626045423817444 .loss/FRt1/detection_loss‚3.016101344939201 1loss/SRqqt2/detection_loss‚1.6857055866469939 1loss/SRqqt4/detection_loss‚1.9249376841682069 %loss/total_loss‚44.55879357663934 epoch‚7 2loss/SRqqt3/assignment_loss‚1.1283067285963526 /loss/FRt2/detection_loss‚2.1535527361763847 /loss/FRt3/detection_loss‚1.6485013342092907 1loss/SRqqt3/detection_loss‚0.9512449642260598 /loss/FRt4/assignment_loss‚3.299036607698158 0loss/FRt1/assignment_loss‚4.4654048488986104 .loss/FBt4/detection_loss‚2.937276909748713÷Òù\Ã#Ö .loss/FRt3/assignment_loss‚2.05558074655987 trainer/global_step‚9299 0loss/FBt1/assignment_loss‚0.5792445898469951 1loss/FBt4/assignment_loss‚0.18104832557340464 .loss/FRt4/detection_loss‚2.280393601981578 1loss/SRqqt1/detection_loss‚2.4027593380365615 /loss/FBt2/detection_loss‚2.1110439234309726 _runtime‚7381.467419139 _step‚764 0loss/FRt2/assignment_loss‚2.9050712744394938 2loss/SRqqt2/assignment_loss‚1.2913358476944268 /loss/FBt2/assignment_loss‚0.719354274786181 /loss/FBt3/assignment_loss‚0.255948271968622 /loss/FBt1/detection_loss‚2.5219691263304815 /loss/FBt3/detection_loss‚0.6579934500964979 $ _timestamp‚1.788452265995899e+09 2loss/SRqqt1/assignment_loss‚1.9243775182045424 2loss/SRqqt4/assignment_loss‚1.4626045423817444 .loss/FRt1/detection_loss‚3.016101344939201 1loss/SRqqt2/detection_loss‚1.6857055866469939 1loss/SRqqt4/detection_loss‚1.9249376841682069 %loss/total_loss‚44.55879357663934 epoch‚7 2loss/SRqqt3/assignment_loss‚1.1283067285963526 /loss/FRt2/detection_loss‚2.1535527361763847 /loss/FRt3/detection_loss‚1.6485013342092907 1loss/SRqqt3/detection_loss‚0.9512449642260598 /loss/FRt4/assignment_loss‚3.299036607698158 0loss/FRt1/assignment_loss‚4.4654048488986104 .loss/FBt4/detection_loss‚2.937276909748713ü—D©LÄ#‚(:’ ·»æÔÓ»þü gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚189.481' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚37.89619999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¤ŠûeÅ#‚(:Ú ·»æԦъÿ disk.drbd1716.in‚0 network.sent‚ 2.2718784e+07 disk.sda2.out‚0 disk.sda3.out‚283.875 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875# memory_percent‚93.5328722000122 disk.dm-0.in‚0.078125 disk.dm-1.out‚283.875 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0 disk.sda1.in‚0) proc.memory.availableMB‚ 1059.57421875 cpu‚25.334497072362375 proc.cpu.threads‚17* disk./.usagePercent‚20.333040508502233 disk.rbd1.in‚0% disk.nvme0n1.out‚29665.0576171875 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚29665.0576171875 disk.sda.out‚283.875 disk.drbd2266.out‚0 network.recv‚ 2.647092e+06$ disk./.usageGB‚302.9769973754883 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125* proc.memory.percent‚16.001558303833008" disk.dm-0.out‚29665.0576171875 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.rbd1.out‚ 41.427734375" proc.memory.rssMB‚ 2621.6953125hÆ‚ÒÆ#‚(:„ Æ»æÔ§ýý gpu.0.gpu‚43.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.4' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.88 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0erosÇ#‚(:è Æ»æÔ Ù¬€ disk.rbd0.in‚0 disk.sda.out‚ 284.5234375 disk.sda2.in‚0 disk.sda2.out‚0* proc.memory.percent‚16.001558303833008* disk./.usagePercent‚20.333933974796732 disk.drbd2266.out‚0# disk.dm-0.out‚29730.36767578125 disk.sda.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚29730.36767578125 disk.sda3.in‚ 0.01171875 network.sent‚ 2.283491e+07 disk.drbd2266.in‚0 cpu‚25.333880902581043 proc.cpu.threads‚17$ disk./.usageGB‚302.9903106689453 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0# memory_percent‚93.5359239578247) proc.memory.availableMB‚ 1059.07421875 disk.rbd0.out‚0" proc.memory.rssMB‚ 2621.6953125 network.recv‚ 2.667588e+06 disk.rbd1.out‚ 41.576171875 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 284.5234375 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 284.5234375& disk.nvme0n1.out‚29730.36767578125ɵFÔ2È#- _wandbruntime‚7418  _runtime‚7418j3ðÞÇÉ#Á & lr-AdamW/pg2‚1.882183009149568e-05 trainer/global_step‚9349 % _timestamp‚1.7884523028903458e+09 _runtime‚7418.362075759 _step‚765 & lr-AdamW/pg1‚1.882183009149568e-05šx¶›ÌÊ#Æ & lr-AdamW/pg1‚1.882183009149568e-05 & lr-AdamW/pg2‚1.882183009149568e-05 trainer/global_step‚9349 % _timestamp‚1.7884523028903458e+09 _runtime‚7418.362075759 _step‚765ý½Âßž2Ë#- _wandbruntime‚7418  _runtime‚7418†h,Wç'Ì#áO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚550985084976128 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 176399879 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7825890926080 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚543159850412544 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 123916993 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚528933852058624 _runtime‚7418.363259744 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚593158227004928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _step‚766 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚550985084976128 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚79244764 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚528933869097984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7825890926080 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚543159868415488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8209334501376 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚449 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 123916542 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 135822644 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7797082315974 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 312242561 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7884523028912914e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚550985759341568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7825234563584 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 176419917 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7796427878218 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 135822644 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚44671778 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚79244764 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚528933869097984 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7825890926080 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚543159868415488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8209334501376 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QIDeviceStatsMonitor.on_train_batch_start/i./a[nactive_split.small_pool.current‚449 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 123916542 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 135822644 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7797082315974 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 312242561 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 % _timestamp‚1.7884523028912914e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚550985759341568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7825234563584 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 176419917 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7796427878218 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 135822644 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚44671778 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚550985084976128 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 176399879 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7825890926080 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚543159850412544 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 123916993 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚528933852058624 _runtime‚7418.363259744 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚593158227004928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 _step‚766 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚550985084976128þñPS2Î#- _wandbruntime‚7419  _runtime‚7419ûÌYñ&Ï#ëM XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7826710367232 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 312275057 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚79252932 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚44676768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚7419.066668517 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚543216203451392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚536786671979690 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 176418333 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7826710367232 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚543216221454336 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚551042931821568 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 123929245 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 312275057 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚79252934 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8210205976064 % _timestamp‚1.7884523035944288e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 135836658 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 176418333 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚543216221454336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7797898751146 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚543216221454336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7797898751146 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7826710367232 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 312275057 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚79252932 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚44676768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚7419.066668517 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚543216203451392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚457 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚536786671979690 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 176418333 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7826710367232 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚543216221454336 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚551042931821568 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 123929245 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 312275057 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚79252934 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8210205976064 % _timestamp‚1.7884523035944288e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0ÿi &p2Ñ#- _wandbruntime‚7419  _runtime‚7419éþùŽVÒ#Ð /loss/FRt2/assignment_loss‚3.442147347189131 0loss/FBt2/assignment_loss‚0.5975962817668915 /loss/FBt4/detection_loss‚2.5334968864917755 trainer/global_step‚9349 _step‚768 0loss/FBt4/assignment_loss‚1.0515934471040964 0loss/SRqqt3/detection_loss‚0.971879633091804 .loss/FBt2/detection_loss‚2.809657710790634 1loss/SRqqt2/detection_loss‚1.9053686575757132 _runtime‚7419.067737264 2loss/SRqqt1/assignment_loss‚2.1721311369910836 2loss/SRqqt2/assignment_loss‚1.3546019642717309 1loss/SRqqt4/assignment_loss‚1.307226351920332 .loss/FRt2/detection_loss‚2.365648600317183 epoch‚7 0loss/FBt1/assignment_loss‚0.8109919084944273 1loss/FBt3/assignment_loss‚0.20327610394688747 /loss/FRt1/detection_loss‚3.1795169830322267 .loss/FBt1/detection_loss‚2.714715980566465 /loss/FBt3/detection_loss‚0.6915813557444567 % _timestamp‚1.7884523035962582e+09 1loss/SRqqt3/assignment_loss‚1.115577303309692 .loss/FRt4/detection_loss‚2.319400590399037 %loss/total_loss‚45.83491786565209 /loss/FRt1/assignment_loss‚3.426875295639038 /loss/FRt3/assignment_loss‚1.624316440688239 0loss/FRt4/assignment_loss‚3.2325864926628443 /loss/FRt3/detection_loss‚2.0103835376600423 1loss/SRqqt1/detection_loss‚2.5639096684753895 1loss/SRqqt4/detection_loss‚1.4304381875229664¨Ê›­[Ó#Õ /loss/FBt3/detection_loss‚0.6915813557444567 % _timestamp‚1.7884523035962582e+09 1loss/SRqqt3/assignment_loss‚1.115577303309692 .loss/FRt4/detection_loss‚2.319400590399037 %loss/total_loss‚45.83491786565209 /loss/FRt1/assignment_loss‚3.426875295639038 /loss/FRt3/assignment_loss‚1.624316440688239 0loss/FRt4/assignment_loss‚3.2325864926628443 /loss/FRt3/detection_loss‚2.0103835376600423 1loss/SRqqt1/detection_loss‚2.5639096684753895 1loss/SRqqt4/detection_loss‚1.4304381875229664 /loss/FRt2/assignment_loss‚3.442147347189131 0loss/FBt2/assignment_loss‚0.5975962817668915 /loss/FBt4/detection_loss‚2.5334968864917755 trainer/global_step‚9349 _step‚768 0loss/FBt4/assignment_loss‚1.0515934471040964 0loss/SRqqt3/detection_loss‚0.971879633091804 .loss/FBt2/detection_loss‚2.809657710790634 1loss/SRqqt2/detection_loss‚1.9053686575757132 _runtime‚7419.067737264 2loss/SRqqt1/assignment_loss‚2.1721311369910836 2loss/SRqqt2/assignment_loss‚1.3546019642717309 1loss/SRqqt4/assignment_loss‚1.307226351920332 .loss/FRt2/detection_loss‚2.365648600317183 epoch‚7 0loss/FBt1/assignment_loss‚0.8109919084944273 1loss/FBt3/assignment_loss‚0.20327610394688747 /loss/FRt1/detection_loss‚3.1795169830322267 .loss/FBt1/detection_loss‚2.714715980566465€_ƒÕ“Ô#‚(:ˆ Õ»æÔ±šþü gpu.0.gpu‚36.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.482' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8964 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0_:è&rÕ#‚(:ç Õ»æÔÕÇ”ÿ disk.sda1.out‚0 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 network.recv‚ 2.670165e+06 disk.nvme0n1.in‚0.078125( disk.nvme0c0n1.out‚29759.39892578125 disk.sda2.out‚0 cpu‚25.333317995916754$ disk./.usageGB‚302.9984016418457 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.rbd0.in‚0 proc.cpu.threads‚17 disk.sda1.in‚0 disk.drbd2080.out‚0* disk./.usagePercent‚20.334476966776858& disk.nvme0n1.out‚29759.39892578125 disk.drbd2265.in‚0 disk.drbd2266.out‚0* proc.memory.percent‚16.001558303833008 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.sda.out‚ 284.9140625 disk.rbd1.in‚0 disk.rbd1.out‚ 41.576171875$ memory_percent‚93.53313446044922 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 284.9140625 disk.drbd2080.in‚0 disk.sda3.out‚ 284.9140625# disk.dm-0.out‚29759.39892578125& proc.memory.availableMB‚ 1059.53125" proc.memory.rssMB‚ 2621.6953125 network.sent‚ 2.2839134e+07 disk.drbd1716.in‚0 disk.sda3.in‚ 0.01171875¬×#ã“Ö#‚(:ˆ ä»æÔõÿ§ý gpu.0.gpu‚48.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚187.833' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.5666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0GMu×#‚(:ê ä»æÔèøÎÿ network.recv‚ 2.696341e+06 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚29817.64306640625 disk.sda.in‚ 0.01171875$ disk./.usageGB‚302.9911193847656 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.rbd0.out‚0$ memory_percent‚93.53766441345215* disk./.usagePercent‚20.333988248393993 disk.drbd2265.in‚0 disk.sda3.out‚ 285.5078125 disk.sda1.out‚0 disk.sda2.in‚0* proc.memory.percent‚16.001558303833008 disk.drbd2080.in‚0 disk.drbd2266.out‚0 network.sent‚ 2.2957858e+07 disk.dm-1.in‚ 0.01171875 disk.rbd0.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.rbd1.out‚ 41.6083984375& disk.nvme0n1.out‚29817.64306640625 disk.dm-1.out‚ 285.5078125 disk.drbd2266.in‚0 cpu‚25.332779267999598 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.sda.out‚ 285.5078125 disk.sda2.out‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚29817.64306640625 disk.sda3.in‚ 0.01171875( proc.memory.availableMB‚ 1058.7890625 proc.cpu.threads‚17ÎeRÊHØ#‚(:“ ó»æÔÁß›ý gpu.0.gpu‚54.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.737' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.547399999999996 gpu.0.smClock‚1410 gpu.0.memo¸~VryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ô#uyÙ#‚(:î ó»æÔ¿´€ disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.drbd2080.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.drbd2266.in‚0 disk.rbd1.out‚ 41.7021484375 disk.drbd1716.in‚0 disk.drbd2265.in‚0 network.recv‚ 2.700171e+06* disk./.usagePercent‚20.335211964407662 disk.sda.out‚ 286.10546875 disk.drbd2265.out‚0* proc.memory.percent‚16.001558303833008$ disk./.usageGB‚303.0093536376953 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125) proc.memory.availableMB‚ 1059.51953125 disk.dm-1.out‚ 286.10546875$ memory_percent‚93.53320598602295 cpu‚25.332207033517587 disk.sda3.out‚ 286.10546875# disk.dm-0.out‚29858.59228515625 disk.sda1.in‚0& disk.nvme0n1.out‚29858.59228515625 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 proc.cpu.threads‚17 network.sent‚ 2.2963273e+07( disk.nvme0c0n1.out‚29858.59228515625|ŠM72Ú#- _wandbruntime‚7455  _runtime‚7455Z°STÈÛ# $ _timestamp‚1.788452340252158e+09 _runtime‚7455.723745815 _step‚769 ' lr-AdamW/pg1‚1.8176032668455404e-05 ' lr-AdamW/pg2‚1.8176032668455404e-05 trainer/global_step‚9399éÕnÍÜ#Ç _step‚769 ' lr-AdamW/pg1‚1.8176032668455404e-05 ' lr-AdamW/pg2‚1.8176032668455404e-05 trainer/global_step‚9399 $ _timestamp‚1.788452340252158e+09 _runtime‚7455.723745815‡K'-2Ý#- _wandbruntime‚7455  _runtime‚7455V F—ç'Þ#áO QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚545977502354944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7866876731904 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 313869413 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7866220369408 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 136523442 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚44899101 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚44898651 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚604493190134784 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚9399 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 136523442 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 177325931 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚539516993281366 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 177325931 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚79653164 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 136523444 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚553843722724352 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚545977520357888 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8252596162048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8252426246656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 313849373 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7866876731904 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚553844397089792 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 136523444 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚531679075625984 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚531679058586624 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 124551815 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚553843722724352 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚545977520357888 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8252596162048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8252426246656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 313849373 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7866876731904 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚553844397089792 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 136523444 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚531679075625984 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚531679058586624 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 124551815 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚9399 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 136523442 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 177325931 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚539516993281366 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430‚³Áz2à#- _wandbruntime‚7456  _runtime‚7456ú¶Û ñ&á#ëM OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 177364449 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚546033873396736 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 136537458 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚546033855393792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 313881843 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚553901569569792 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚531733979756544 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7838734090554 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚9399 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _step‚771 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚546033855393792 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7867039797248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 177344385 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚531733962717184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7838079652590 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚596302438565888 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 177364449 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 124565001 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 313901909 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚79661332 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8253447616512 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 313901909 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚470 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8253277714432 _runtime‚7456.420020845 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884523409478438e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 313881843 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 136537460 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚539572042369774 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 136537458 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚546033873396736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚553900895191040 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 124564529 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚79661334 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚44903197 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7867696173056 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7867696173056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚596302414625792 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚539572713847098 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚44903667 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 177344385 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944'ðÄõâ#ðM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚44903667 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 177344385 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 177364449 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚546033873396736 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 136537458 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚546033855393792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 313881843 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚553901569569792 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚531733979756544 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7838734090554 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 trainer/global_step‚9399 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _step‚771 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚546033855393792 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7867039797248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 177344385 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚531733962717184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7838079652590 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚596302438565888 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 177364449 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 124565001 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 313901909 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚79661332 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8253447616512 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 313901909 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚470 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8253277714432 _runtime‚7456.420020845 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884523409478438e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 313881843 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 136537460 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚539572042369774 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 136537458 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚546033873396736 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚553900895191040 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 124564529 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚79661334 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚44903197 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7867696173056 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7867696173056 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚596302414625792 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚539572713847098 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401ƒǶã]2ã#- _wandbruntime‚7456  _runtime‚7456mÒ1yYä#Ó 0loss/FRt2/assignment_loss‚3.7693281359142725 1loss/FBt3/assignment_loss‚0.14954811923099973 /loss/FRt3/detection_loss‚1.7156394078415267 2loss/SRqqt2/assignment_loss‚1.6908918962221253 2loss/SRqqt3/assignment_loss‚1.0010236224506872 0loss/FBt1/assignment_loss‚0.6119501190880935 0loss/FBt2/assignment_loss‚0.3057894645258209 0loss/FBt4/assignment_loss‚0.1607422659144504 0loss/SRqqt4/detection_loss‚1.565051003069174 /loss/FBt2/detection_loss‚2.6220492631196977 epoch‚7 1loss/SRqqt1/detection_loss‚2.6768502249167514 /loss/FBt3/detection_loss‚0.7097703268334857 &loss/total_loss‚45.366588310043554 _step‚772 -loss/FRt2/detection_loss‚2.31588714255227 1loss/SRqqt3/detection_loss‚1.0026263549797616 $ _timestamp‚1.788452340949617e+09 2loss/SRqqt1/assignment_loss‚1.8832176648653471 2loss/SRqqt4/assignment_loss‚1.2592692925426803 .loss/FBt4/detection_loss‚4.794528663158417 _runtime‚7456.421127183 .loss/FBt1/detection_loss‚1.854451741774877 trainer/global_step‚9399 0loss/FRt3/assignment_loss‚1.5700512710882693 0loss/FRt4/assignment_loss‚2.8210581312788294 1loss/SRqqt2/detection_loss‚1.9290380559184335 0loss/FRt1/assignment_loss‚3.8687084173334054 .loss/FRt1/detection_loss‚2.935863396217083 .loss/FRt4/detection_loss‚2.153254329207096}£_m^å#Ø .loss/FRt1/detection_loss‚2.935863396217083 .loss/FRt4/detection_loss‚2.153254329207096 0loss/FRt2/assignment_loss‚3.7693281359142725 1loss/FBt3/assignment_loss‚0.14954811923099973 /loss/FRt3/detection_loss‚1.7156394078415267 2loss/SRqqt2/assignment_loss‚1.6908918962221253 2loss/SRqqt3/assignment_loss‚1.0010236224506872 0loss/FBt1/assignment_loss‚0.6119501190880935 0loss/FBt2/assignment_loss‚0.3057894645258209 0loss/FBt4/assignment_loss‚0.1607422659144504 0loss/SRqqt4/detection_loss‚1.565051003069174 /loss/FBt2/detection_loss‚2.6220492631196977 epoch‚7 1loss/SRqqt1/detection_loss‚2.6768502249167514 /loss/FBt3/detection_loss‚0.7097703268334857 &loss/total_loss‚45.366588310043554 _step‚772 -loss/FRt2/detection_loss‚2.31588714255227 1loss/SRqqt3/detection_loss‚1.0026263549797616 $ _timestamp‚1.788452340949617e+09 2loss/SRqqt1/assignment_loss‚1.8832176648653471 2loss/SRqqt4/assignment_loss‚1.2592692925426803 .loss/FBt4/detection_loss‚4.794528663158417 _runtime‚7456.421127183 .loss/FBt1/detection_loss‚1.854451741774877 trainer/global_step‚9399 0loss/FRt3/assignment_loss‚1.5700512710882693 0loss/FRt4/assignment_loss‚2.8210581312788294 1loss/SRqqt2/detection_loss‚1.9290380559184335 0loss/FRt1/assignment_loss‚3.8687084173334054„vuqš“æ#‚(:ˆ ‚¼æÔŽ¨žþ gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.978' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5956 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0KN‡/mç#‚(:â ‚¼æÔõŽäÿ network.sent‚ 2.2973207e+07 disk.drbd2080.out‚0! disk.dm-0.out‚29912.951171875 disk.sda1.out‚0 disk.rbd1.in‚0 proc.cpu.threads‚17 disk.drbd2265.in‚0 disk.rbd1.out‚ 41.8505859375 cpu‚25.33161350516957 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.sda3.out‚ 286.7734375 disk.sda1.in‚0 disk.sda2.in‚0* proc.memory.percent‚16.001558303833008& disk.nvme0c0n1.out‚29912.951171875 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.sda2.out‚0* disk./.usagePercent‚20.335647177215872$ disk./.usageGB‚303.0158386230469 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.dm-0.in‚0.078125$ memory_percent‚93.53513717651367 network.recv‚ 2.711879e+06 disk.dm-1.out‚ 286.7734375 disk.drbd2266.in‚0 disk.nvme0n1.in‚0.078125$ disk.nvme0n1.out‚29912.951171875' proc.memory.availableMB‚ 1059.203125" proc.memory.rssMB‚ 2621.6953125 disk.drbd2266.out‚0 disk.sda.out‚ 286.7734375ü4àè#‚(:’ ‘¼æÔ’‰–ý gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚172.531' gpu.0.enforcedPowerLimitWatts‚500.0( gpu.0.powerPercent‚34.50620000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ÿò®né#‚(:ã ‘¼æÔ»…Œ€ disk.sda1.in‚0* disk./.usagePercent‚20.336247514883663 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0$ disk.nvme0n1.out‚29951.197265625 disk.dm-0.in‚0.078125 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0" proc.memory.rssMB‚ 2621.6953125* proc.memory.percent‚16.001558303833008 network.sent‚ 2.3081142e+07 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17% disk./.usageGB‚303.02478408813477 disk.dm-1.out‚ 287.265625 disk.drbd1716.in‚0! disk.dm-0.out‚29951.197265625) proc.memory.availableMB‚ 1059.06640625 disk.sda3.out‚ 287.265625 disk.drbd2266.out‚0 disk.sda2.out‚0& disk.nvme0c0n1.out‚29951.197265625 network.recv‚ 2.717286e+06 disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.rbd1.in‚0$ memory_percent‚93.53597164154053 cpu‚25.331049272600193 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 287.265625 disk.sda2.in‚0 disk.rbd1.out‚ 41.8505859375K™b¡2ê#- _wandbruntime‚7492  _runtime‚7492¸&±8Çë#Á & lr-AdamW/pg1‚1.753964794169447e-05 & lr-AdamW/pg2‚1.753964794169447e-05 trainer/global_step‚9449 % _timestamp‚1.7884523773272867e+09 _runtime‚7492.798999066 _step‚773ÃÊx+Ìì#Æ _step‚773 & lr-AdamW/pg1‚1.753964794169447e-05 & lr-AdamW/pg2‚1.753964794169447e-05 trainer/global_step‚9449 % _timestamp‚1.7884523773272867e+09 _runtime‚7492.798999066…°X»2í#- _wandbruntime‚7492  _runtime‚7492¡ê¥Áç'î#áO TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚542302363671658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚556703034838016 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 _runtime‚7492.799986299 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚556702360472576 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚548795172300288 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7907862537728 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 178272021 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚80061564 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 137224244 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7907862537728 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚599323300771328 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 178251983 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚607618993808384 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7907206175232 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚444 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚45125741 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7878098557034 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 137224244 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 125187305 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚534424265114624 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚774 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7907206175232 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 137224242 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚9449 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚534424282153984 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 315476225 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 137224242 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚607619187663872 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0G–Ñ Ëï#æO ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚607618993808384 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7907206175232 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚444 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚45125741 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7878098557034 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 137224244 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 125187305 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚534424265114624 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚774 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7907206175232 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 137224242 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚9449 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚534424282153984 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 315476225 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 137224242 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚607619187663872 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚542302363671658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚556703034838016 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚446 _runtime‚7492.799986299 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚556702360472576 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚548795172300288 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7907862537728 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 178272021 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚80061564 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 137224244 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7907862537728 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚599323300771328 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 178251983†Ÿ{›O2ð#- _wandbruntime‚7493  _runtime‚74936,2Fð&ñ#êM XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚607681517122560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7879569429962 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚45130751 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚542358084237182 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 _runtime‚7493.505506439 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 178270437 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8296565613568 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7908681978880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 315528761 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 137238258 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 $ _timestamp‚1.788452378033406e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚9449 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚45130314 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8296720835584 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚542358755714506 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚556760207318016 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚599384951508992 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚80069734 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7908025603072 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚607681696284672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7878914991998 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 315508695 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 137238260 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 315528761 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 137238260 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 178290501 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚556760207318016 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7908681978880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚775 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 178270437 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚548851507336192 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚534479169245184 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚548851525339136 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 137238258 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 125200046 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7908025603072 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚548851525339136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚534479186284544 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 93260349445°»Lõ&ò#ïM H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 315528761 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 137238258 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 $ _timestamp‚1.788452378033406e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚9449 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚45130314 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8296720835584 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚542358755714506 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚556760207318016 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚599384951508992 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚80069734 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7908025603072 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚607681696284672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7878914991998 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 315508695 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 137238260 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 315528761 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 137238260 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 178290501 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚556760207318016 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7908681978880 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚775 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 178270437 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚548851507336192 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚534479169245184 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚548851525339136 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 137238258 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 125200046 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7908025603072 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚548851525339136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚534479186284544 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚607681517122560 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7879569429962 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚45130751 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚542358084237182 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 _runtime‚7493.505506439 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 178270437 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8296565613568 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7908681978880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424‡ë 2ó#- _wandbruntime‚7493  _runtime‚7493eÔ¸Xô#Ò /loss/FBt2/detection_loss‚1.6963223318258922 /loss/FBt4/detection_loss‚2.6750069005148753 /loss/FRt2/assignment_loss‚3.011837661266327 1loss/FBt1/assignment_loss‚0.31313826858500704 .loss/FRt1/detection_loss‚2.774600069312488 /loss/FRt4/assignment_loss‚3.196803274678021 0loss/SRqqt4/detection_loss‚1.881626552140171 %loss/total_loss‚44.43305874847229 epoch‚7 trainer/global_step‚9449 0loss/FRt1/assignment_loss‚4.4741089414147766 0loss/FRt3/assignment_loss‚1.4962257611751557 0loss/FBt4/assignment_loss‚0.5731559748362217 1loss/SRqqt2/detection_loss‚1.7897693467957358 -loss/FBt1/detection_loss‚2.94172170261542 _runtime‚7493.507013604 2loss/SRqqt3/assignment_loss‚0.9871727447842428 0loss/FBt3/assignment_loss‚0.2766242470089348 0loss/SRqqt1/detection_loss‚2.707340506406931 _step‚776 0loss/FBt2/assignment_loss‚0.7046739828462374 % _timestamp‚1.7884523780351882e+09 2loss/SRqqt1/assignment_loss‚2.4220450090674253 2loss/SRqqt2/assignment_loss‚1.2723180833122423 /loss/FRt2/detection_loss‚2.0971795374155047 /loss/FBt3/detection_loss‚0.5895633881147473 2loss/SRqqt4/assignment_loss‚1.0986529076048597 /loss/FRt4/detection_loss‚2.5715716459402223 1loss/SRqqt3/detection_loss‚1.1055833500924341 .loss/FRt3/detection_loss‚1.776016560718417‰~xý]õ#× % _timestamp‚1.7884523780351882e+09 2loss/SRqqt1/assignment_loss‚2.4220450090674253 2loss/SRqqt2/assignment_loss‚1.2723180833122423 /loss/FRt2/detection_loss‚2.0971795374155047 /loss/FBt3/detection_loss‚0.5895633881147473 2loss/SRqqt4/assignment_loss‚1.0986529076048597 /loss/FRt4/detection_loss‚2.5715716459402223 1loss/SRqqt3/detection_loss‚1.1055833500924341 .loss/FRt3/detection_loss‚1.776016560718417 /loss/FBt2/detection_loss‚1.6963223318258922 /loss/FBt4/detection_loss‚2.6750069005148753 /loss/FRt2/assignment_loss‚3.011837661266327 1loss/FBt1/assignment_loss‚0.31313826858500704 .loss/FRt1/detection_loss‚2.774600069312488 /loss/FRt4/assignment_loss‚3.196803274678021 0loss/SRqqt4/detection_loss‚1.881626552140171 %loss/total_loss‚44.43305874847229 epoch‚7 trainer/global_step‚9449 0loss/FRt1/assignment_loss‚4.4741089414147766 0loss/FRt3/assignment_loss‚1.4962257611751557 0loss/FBt4/assignment_loss‚0.5731559748362217 1loss/SRqqt2/detection_loss‚1.7897693467957358 -loss/FBt1/detection_loss‚2.94172170261542 _runtime‚7493.507013604 2loss/SRqqt3/assignment_loss‚0.9871727447842428 0loss/FBt3/assignment_loss‚0.2766242470089348 0loss/SRqqt1/detection_loss‚2.707340506406931 _step‚776 0loss/FBt2/assignment_loss‚0.7046739828462374ˆÍúξ“ö#‚(:ˆ  ¼æÔ“‰ý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.409' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0,çŽ^÷#‚(:Ó  ¼æÔ¡®¦ÿ network.recv‚ 2.722267e+06 disk.sda.out‚288 disk.sda2.out‚0( disk.nvme0c0n1.out‚30003.60693359375 network.sent‚ 2.3087563e+07 disk.rbd1.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2621.6953125& disk.nvme0n1.out‚30003.60693359375 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2265.out‚0 cpu‚25.330528026462577 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.out‚288 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0 disk.drbd2266.out‚0# disk.dm-0.out‚30003.60693359375 proc.cpu.threads‚17* proc.memory.percent‚16.001558303833008 disk.sda3.out‚288) disk./.usagePercent‚20.33717707824049% disk./.usageGB‚303.03863525390625 disk.rbd1.out‚ 42.4677734375 disk.nvme0n1.in‚0.078125 disk.sda2.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.dm-0.in‚0.078125$ memory_percent‚93.53616237640381) proc.memory.availableMB‚ 1059.03515625Pµ…$“ø#‚(:ˆ ¯¼æÔ‡³ñü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚190.619' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.1238 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0$Mï iù#‚(:Þ ¯¼æÔæ׫ÿ disk.dm-0.in‚0.078125 disk.rbd1.out‚ 42.5615234375 network.recv‚ 2.727414e+06 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚288.5625 disk.sda3.in‚ 0.01171875" disk.dm-0.out‚30051.9814453125 disk.sda2.in‚0 disk.sda2.out‚0$ memory_percent‚93.53616237640381 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125" proc.memory.rssMB‚ 2621.6953125* proc.memory.percent‚16.001558303833008 network.sent‚ 2.3195218e+07 disk.drbd2080.out‚0 disk.sda3.out‚288.5625 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.rbd1.in‚0 cpu‚25.329970002226375 proc.cpu.threads‚17# disk./.usageGB‚303.049503326416 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚30051.9814453125 disk.rbd0.in‚0 disk.rbd0.out‚0% disk.nvme0n1.out‚30051.9814453125 disk.drbd1716.out‚0 disk.sda.out‚288.5625 disk.sda1.out‚0) proc.memory.availableMB‚ 1059.03515625* disk./.usagePercent‚20.337906443705542 disk.drbd2080.in‚0Ъƒº“ú#‚(:ˆ ¾¼æÔÃæÓý gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.409' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0œÐh~mû#‚(:â ¾¼æÔÉøÿ disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125& proc.memory.availableMB‚ 1059.03125 network.recv‚ 2.732664e+06 disk.sda.in‚ 0.01171875 disk.sda.out‚ 289.2109375 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0$ memory_percent‚93.53618621826172 network.sent‚ 2.3201899e+07* disk./.usagePercent‚20.338553630752102$ disk./.usageGB‚303.0591468811035 disk.rbd0.out‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 289.2109375 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.drbd2265.in‚0& disk.nvme0c0n1.out‚30115.337890625" proc.memory.rssMB‚ 2621.6953125 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875* proc.memory.percent‚16.001558303833008 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda3.out‚ 289.2109375 disk.drbd2266.out‚0 disk.rbd0.in‚0! disk.dm-0.out‚30115.337890625 disk.sda2.out‚0$ disk.nvme0n1.out‚30115.337890625 cpu‚25.329410271025864 disk.rbd1.out‚ 42.6240234375 disk.nvme0n1.in‚0.078125R¥2ü#- _wandbruntime‚7530  _runtime‚7530 ?…!Éý#à ' lr-AdamW/pg1‚1.6912808532836098e-05 ' lr-AdamW/pg2‚1.6912808532836098e-05 trainer/global_step‚9499 % _timestamp‚1.7884524147103448e+09 _runtime‚7530.181993783 _step‚777.ÞØÎþ#È ' lr-AdamW/pg1‚1.6912808532836098e-05 ' lr-AdamW/pg2‚1.6912808532836098e-05 trainer/global_step‚9499 % _timestamp‚1.7884524147103448e+09 _runtime‚7530.181993783 _step‚777‰|%…ï2ÿ#- _wandbruntime‚7530  _runtime‚75303&9ú{€$áO E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚559561672586240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚45354352 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8339043469824 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚9499 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 125824316 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_start/5«Ül#allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 179178035 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚80469966 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚602405837654528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚545089077016182 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 317103077 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚551612806239744 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8338871457280 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 137925042 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚551612806239744 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 137925044 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚45354820 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚7530.183108136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚551612824242688 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 137925042 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7948848343552 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 179178035 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚545088405539066 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _step‚778 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7948848343552 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚537169471642624 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 125824786 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚468 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7919588334198 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 179198073 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚610744905064448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚602405861594624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 % _timestamp‚1.7884524147111723e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚610744709111808 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚559560998220800 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 317123117 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚559560998220800 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚537169488681984 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 179198073 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7948191981056 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚551612824242688 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚559561672586240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7948191981056 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430¨ÎOmì'$æO F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 % _timestamp‚1.7884524147111723e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚610744709111808 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚559560998220800 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 317123117 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚559560998220800 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚537169488681984 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 179198073 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 7948191981056 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚551612824242688 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚559561672586240 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7948191981056 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚559561672586240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚45354352 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8339043469824 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚9499 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 125824316 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 179178035 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚80469966 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚602405837654528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚545089077016182 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 317103077 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚551612806239744 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8338871457280 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 137925042 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚551612806239744 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 137925044 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚45354820 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚7530.183108136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚551612824242688 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 137925042 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7948848343552 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 179178035 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚545088405539066 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _step‚778 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 7948848343552 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚537169471642624 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 125824786 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚468 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7919588334198 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 179198073 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚610744905064448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚602405861594624 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376ŠN\2‚$- _wandbruntime‚7530  _runtime‚7530…þÊ:ñ&ƒ$ëM H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 317155613 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚45359380 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚545144126104590 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 125837056 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚551669159278592 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚551669177281536 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 137939058 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚610807400830976 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚537224375773184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 137939058 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7949667784704 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 317155613 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 137939060 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚779 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7949667784704 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8339888498688 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚551669177281536 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 179216553 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚545144797581914 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚9499 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚610807206988800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚602467512332288 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7920404769370 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7919750331406 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚602467488392192 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7949011408896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚559618170687488 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 137939060 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚537224392812544 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7949011408896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚80478134 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚80478132 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚7530.878529617 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 179216553 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 179196489 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 317135547 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884524154064336e+09 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 317135547 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944ºØ±b „$ðM FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚9499 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚610807206988800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚602467512332288 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7920404769370 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 7919750331406 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚602467488392192 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 7949011408896 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚559618170687488 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 137939060 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚537224392812544 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 7949011408896 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚80478134 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚80478132 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚7530.878529617 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 179216553 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 179196489 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 317135547 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.78845lÂÉÒ”24154064336e+09 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 317135547 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 317155613 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚45359380 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚545144126104590 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 125837056 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚551669159278592 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚551669177281536 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 137939058 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚610807400830976 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚537224375773184 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 137939058 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 7949667784704 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 317155613 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 137939060 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚779 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7949667784704 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8339888498688 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚551669177281536 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 179216553 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚545144797581914 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2‹àkF2…$- _wandbruntime‚7530  _runtime‚7530šÃ¦•X†$Ò 2loss/SRqqt2/assignment_loss‚1.1295021941483605 1loss/SRqqt4/detection_loss‚1.9833070594579616 2loss/SRqqt4/assignment_loss‚1.2492522699560256 -loss/FBt1/detection_loss‚2.70568631704037 /loss/FBt3/detection_loss‚0.7577249238219781 /loss/FRt1/detection_loss‚2.9970942040284476 /loss/FRt2/detection_loss‚2.2543263069607993 0loss/FRt1/assignment_loss‚3.9500166793664295 1loss/SRqqt3/assignment_loss‚1.287681296645828 /loss/FBt2/detection_loss‚1.5798594132065773 1loss/FBt4/assignment_loss‚0.17727435088558455 .loss/FRt4/detection_loss‚2.421339781043377 1loss/SRqqt3/detection_loss‚1.0555023673702688 trainer/global_step‚9499 _step‚780 0loss/FRt4/assignment_loss‚3.1790693174017237 2loss/SRqqt1/assignment_loss‚1.8067606035055537 0loss/FBt2/assignment_loss‚0.5249917311011814 _runtime‚7530.879685148 0loss/SRqqt1/detection_loss‚2.709170727115689 epoch‚7 0loss/SRqqt2/detection_loss‚1.859445204921797 0loss/FBt1/assignment_loss‚0.5672995617107447 /loss/FRt3/detection_loss‚1.5983043877733132 .loss/FBt4/detection_loss‚2.562987949166979 %loss/total_loss‚43.00724775041716 % _timestamp‚1.7884524154081662e+09 0loss/FBt3/assignment_loss‚0.1668879680228052 /loss/FRt2/assignment_loss‚2.615304298021577 0loss/FRt3/assignment_loss‚1.8684588377437894øÉ-Ù]‡$× 0loss/FBt1/assignment_loss‚0.5672995617107447 /loss/FRt3/detection_loss‚1.5983043877733132 .loss/FBt4/detection_loss‚2.562987949166979 %loss/total_loss‚43.00724775041716 % _timestamp‚1.7884524154081662e+09 0loss/FBt3/assignment_loss‚0.1668879680228052 /loss/FRt2/assignment_loss‚2.615304298021577 0loss/FRt3/assignment_loss‚1.8684588377437894 2loss/SRqqt2/assignment_loss‚1.1295021941483605 1loss/SRqqt4/detection_loss‚1.9833070594579616 2loss/SRqqt4/assignment_loss‚1.2492522699560256 -loss/FBt1/detection_loss‚2.70568631704037 /loss/FBt3/detection_loss‚0.7577249238219781 /loss/FRt1/detection_loss‚2.9970942040284476 /loss/FRt2/detection_loss‚2.2543263069607993 0loss/FRt1/assignment_loss‚3.9500166793664295 1loss/SRqqt3/assignment_loss‚1.287681296645828 /loss/FBt2/detection_loss‚1.5798594132065773 1loss/FBt4/assignment_loss‚0.17727435088558455 .loss/FRt4/detection_loss‚2.421339781043377 1loss/SRqqt3/detection_loss‚1.0555023673702688 trainer/global_step‚9499 _step‚780 0loss/FRt4/assignment_loss‚3.1790693174017237 2loss/SRqqt1/assignment_loss‚1.8067606035055537 0loss/FBt2/assignment_loss‚0.5249917311011814 _runtime‚7530.879685148 0loss/SRqqt1/detection_loss‚2.709170727115689 epoch‚7 0loss/SRqqt2/detection_loss‚1.859445204921797ŒÎžˆ$‚(:“ ͼæÔ×™›ý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚138.948' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.789600000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ž…"Bq‰$‚(:æ ͼæÔÝþ’ÿ disk.sda1.out‚0 cpu‚25.328829414195308 proc.cpu.threads‚17! disk.dm-0.out‚30157.654296875 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.sda1.in‚0 disk.sda2.in‚0* proc.memory.percent‚16.001558303833008 disk.rbd1.out‚ 42.7099609375 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 289.55078125 disk.dm-0.in‚0.078125* disk./.usagePercent‚20.339152176367154% disk./.usageGB‚303.06806564331055 disk.drbd1716.out‚0 disk.sda3.in‚ 0.01171875 network.sent‚ 2.3207178e+07 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.sda.out‚ 289.55078125$ disk.nvme0n1.out‚30157.654296875 disk.dm-1.out‚ 289.55078125 disk.sda2.out‚0$ memory_percent‚93.53656768798828" proc.memory.rssMB‚ 2621.6953125 disk.sda.in‚ 0.01171875 network.recv‚ 2.736131e+06 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0& proc.memory.availableMB‚ 1058.96875 disk.rbd0.out‚0 disk.drbd2265.out‚0& disk.nvme0c0n1.out‚30157.654296875’ÛI—jŠ$‚(:ß ܼæÔŸ³¡ÿ disk.sda3.in‚ 0.01171875 disk.dm-0.out‚30206.32421875 disk.sda.out‚ 290.171875 disk.drbd2265.out‚0 disk.drbd2080.out‚0% disk.nvme0c0n1.out‚30206.32421875 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.dm-1.out‚ 290.171875$ memory_percent‚93.53806972503662* proc.memory.percent‚16.001558303833008 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 290.171875 disk.sda1.out‚0# disk.nvme0n1.out‚30206.32421875 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0 disk.sda2.in‚0) proc.memory.availableMB‚ 1058.72265625 disk.dm-0.in‚0.078125 disk.sda1.in‚0" proc.memory.rssMB‚ 2621.6953125 network.sent‚ 2.3316071e+07 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.rbd0.in‚0 disk.drbd1716.in‚0 network.recv‚ 2.742548e+06 disk.drbd2080.in‚0 disk.rbd1.out‚ 42.8583984375 disk.nvme0n1.in‚0.078125 cpu‚25.328277513775955 proc.cpu.threads‚17) disk./.usagePercent‚20.33968390401577% disk./.usageGB‚303.07598876953125 disk.drbd2265.in‚0ùdu“‹$‚(:ˆ ܼæÔí雩 gpu.0.gpu‚37.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.409' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0½ ç2Œ$- _wandbruntime‚7567  _runtime‚7567ÂÄëÉ$à ' lr-AdamW/pg2‚1.6295645074273585e-05 trainer/global_step‚9549 % _timestamp‚1.7884524518509955e+09 _runtime‚7567.322602698 _step‚781 ' lr-AdamW/pg1‚1.6295645074273585e-05·^%ÕÎŽ$È trainer/global_step‚9549 % _timestamp‚1.7884524518509955e+09 _runtime‚7567.322602698 _step‚781 ' lr-AdamW/pg1‚1.6295645074273585e-05 ' lr-AdamW/pg2‚1.6295645074273585e-05Éú9­2$- _wandbruntime‚7567  _runtime‚7567Û´& ç'$áO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚562419635969024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚562420310334464 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7989177786880 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8381968768512 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 180124125 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚554430458182144 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884524518517354e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚45582151 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 138625842 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚45582608 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8382138683904 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚782 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚562420310334464 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚554430476185088 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 180104087 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚562419635969024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚554430458182144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚539914678170624 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚554430476185088 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚547875118883590 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 138625844 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚80878366 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚547874447406474 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 126460515 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7960423673606 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7959769235850 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7989834149376 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚539914695209984 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 138625844(®ÈŸ5%‘$æO O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 7959769235850 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 7989834149376 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚539914695209984 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 138625844 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚562419635969024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚562420310334464 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 7989177786880 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8381968768512 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 180124125 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚554430458182144 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884524518517354e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚45582151 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 138625842 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚45582608 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8382138683904 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚782 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚562420310334464 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚554430476185088 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 180104087 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚562419635969024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚554430458182144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚539914678170624 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚554430476185088 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚547875118883590 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 138625844 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.laÊÛœ··rge_pool.allocated‚80878366 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚547874447406474 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 126460515 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 7960423673606Ž†¿ê§2’$- _wandbruntime‚7568  _runtime‚7568ØðÒ·ñ&“$ëM ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚554486829223936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚45587170 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚562477482814464 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7961240108778 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚80886532 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 138639860 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚562476808435712 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 180142605 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 180122541 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚605550025275392 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 138639860 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚613932864566784 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚539969582301184 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚554486811220992 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚462 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8382996610560 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7990653590528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 138639858 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 126473704 trainer/global_step‚9549 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚554486829223936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 318762399 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 180122541 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚613933045826048 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚605550049215488 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 126473242 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚547930839449322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚547930167971998 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 318782465 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 % _timestamp‚1.7884524525471616e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚80886534 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚539969599340544 _runtime‚7568.019419915 _step‚783yÚøñö&”$ðM UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 138639860 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚562476808435712 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 180142605 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 180122541 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚605550025275392 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 138639860 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚613932864566784 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚539969582301184 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚554486811220992 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚462 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8382996610560 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 7990653590528 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 138639858 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 126473704 trainer/global_step‚9549 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚554486829223936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 318762399 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 180122541 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚613933045826048 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚605550049215488 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 126473242 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚547930839449322 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚547930167971998 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 318782465 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 % _timestamp‚1.7884524525471616e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚80886534 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚539969599340544 _runtime‚7568.019419915 _step‚783 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚554486829223936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚45587170 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚562477482814464 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 7961240108778 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚80886532 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0}~€½2•$- _wandbruntime‚7568  _runtime‚75689O'Ã\–$Ö 1loss/SRqqt3/detection_loss‚1.1219631387468647 .loss/FBt1/detection_loss‚2.282189538081487 0loss/FRt4/assignment_loss‚2.6973250055754625 1loss/SRqqt2/detection_loss‚1.9307251523952096 /loss/FBt2/detection_loss‚1.9160021408037706 0loss/FBt3/detection_loss‚0.48314903764804074 epoch‚7 0loss/FRt3/assignment_loss‚2.1583831884644247 2loss/SRqqt3/assignment_loss‚1.0792532521710896 1loss/SRqqt4/detection_loss‚1.6293777044405018 $loss/total_loss‚46.2928990697731 /loss/FRt2/detection_loss‚2.4671991552625383 1loss/SRqqt1/detection_loss‚2.5575520869563606 /loss/FRt1/assignment_loss‚4.878298211097717 2loss/SRqqt2/assignment_loss‚1.7314373078014778 2loss/SRqqt4/assignment_loss‚1.4536310909152554 1loss/FBt1/assignment_loss‚0.42360747115079295 .loss/FBt4/detection_loss‚3.729999383290609 trainer/global_step‚9549 _step‚784 2loss/SRqqt1/assignment_loss‚1.7351530369590311 0loss/FBt2/assignment_loss‚0.6083175922659311 1loss/FBt3/assignment_loss‚0.20468888204919075 0loss/FBt4/assignment_loss‚0.5524115612109503 /loss/FRt3/detection_loss‚1.9791703396222808 .loss/FRt4/detection_loss‚2.115071776426501 % _timestamp‚1.7884524525490823e+09 _runtime‚7568.020600563 /loss/FRt2/assignment_loss‚3.568143258289415 /loss/FRt1/detection_loss‚2.9898497581481935}£Œa—$Û 2loss/SRqqt1/assignment_loss‚1.7351530369590311 0loss/FBt2/assignment_loss‚0.6083175922659311 1loss/FBt3/assignment_loss‚0.20468888204919075 0loss/FBt4/assignment_loss‚0.5524115612109503 /loss/FRt3/detection_loss‚1.9791703396222808 .loss/FRt4/detection_loss‚2.115071776426501 % _timestamp‚1.7884524525490823e+09 _runtime‚7568.020600563 /loss/FRt2/assignment_loss‚3.568143258289415 /loss/FRt1/detection_loss‚2.9898497581481935 1loss/SRqqt3/detection_loss‚1.1219631387468647 .loss/FBt1/detection_loss‚2.282189538081487 0loss/FRt4/assignment_loss‚2.6973250055754625 1loss/SRqqt2/detection_loss‚1.9307251523952096 /loss/FBt2/detection_loss‚1.9160021408037706 0loss/FBt3/detection_loss‚0.48314903764804074 epoch‚7 0loss/FRt3/assignment_loss‚2.1583831884644247 2loss/SRqqt3/assignment_loss‚1.0792532521710896 1loss/SRqqt4/detection_loss‚1.6293777044405018 $loss/total_loss‚46.2928990697731 /loss/FRt2/detection_loss‚2.4671991552625383 1loss/SRqqt1/detection_loss‚2.5575520869563606 /loss/FRt1/assignment_loss‚4.878298211097717 2loss/SRqqt2/assignment_loss‚1.7314373078014778 2loss/SRqqt4/assignment_loss‚1.4536310909152554 1loss/FBt1/assignment_loss‚0.42360747115079295 .loss/FBt4/detection_loss‚3.729999383290609 trainer/global_step‚9549 _step‚7840s¬Ãž˜$‚(:“ ë¼æÔ„À¡ý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.834' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.766799999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0iæìØo™$‚(:ä ë¼æÔŸ¶—€ cpu‚25.32775514143843 disk.dm-0.out‚30249.83984375 disk.nvme0n1.in‚0.078125 disk.nvme0c0n1.in‚0.078125) proc.memory.availableMB‚ 1058.93359375 disk.drbd2265.in‚0 disk.drbd2080.in‚0 network.recv‚ 2.745718e+06 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.dm-1.out‚ 290.77734375 disk.drbd1716.out‚0 disk.sda3.in‚ 0.01171875* proc.memory.percent‚16.001558303833008$ disk./.usageGB‚303.0824317932129 disk.rbd0.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.rbd1.in‚0# disk.nvme0n1.out‚30249.83984375 disk.drbd2266.in‚0 disk.drbd2266.out‚0 network.sent‚ 2.3321323e+07 disk.sda2.in‚0 disk.drbd1716.in‚0$ memory_percent‚93.53678226470947* disk./.usagePercent‚20.340116300741105 disk.rbd0.in‚0 disk.sda2.out‚0% disk.nvme0c0n1.out‚30249.83984375" proc.memory.rssMB‚ 2621.6953125 disk.sda.out‚ 290.77734375 disk.rbd1.out‚ 43.3115234375 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 290.77734375 proc.cpu.threads‚17 disk.dm-1.in‚ 0.01171875õí“š$‚(:ˆ ú¼æÔ§ûÓý gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.229' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4458 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0UÔŽs›$‚(:è ú¼æÔ–¼Ïÿ disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda.out‚ 291.55078125 cpu‚25.32721213510187* proc.memory.percent‚16.001558303833008 disk.dm-1.out‚ 291.55078125 network.recv‚ 2.75267e+06 disk.sda2.out‚0% disk.nvme0n1.out‚30304.2138671875 disk.drbd2080.in‚0 disk.sda.in‚ 0.01171875) disk./.usagePercent‚20.34051464846438 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚30304.2138671875$ disk./.usageGB‚303.0883674621582 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0) proc.memory.availableMB‚ 1058.93359375" proc.memory.rssMB‚ 2621.6953125 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 291.55078125$ memory_percent‚93.53678226470947 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.drbd2265.in‚0" disk.dm-0.out‚30304.2138671875 network.sent‚ 2.3430532e+07 disk.rbd1.out‚ 43.7333984375 disk.drbd2266.out‚0 disk.dm-1.in‚ 0.01171875 disk.sda1.in‚0ÿQ« žœ$‚(:“ ‰½æÔ¼…ñü gpu.0.gpu‚39.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.737' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.547399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Öè±u$‚(:ê ‰½æÔÝÞÿ disk.rbd1.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0 disk.rbd1.out‚ 43.7333984375 disk.dm-1.out‚ 292.19921875 disk.drbd2265.in‚0$ memory_percent‚93.53678226470947 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚30345.7373046875 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2621.6953125) disk./.usagePercent‚20.34172146798079 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚ 292.19921875 cpu‚25.32663816852099 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 292.19921875 disk.sda1.in‚0 disk.sda2.out‚0% disk.nvme0n1.out‚30345.7373046875 disk.drbd2265.out‚0% disk./.usageGB‚303.10634994506836 disk.nvme0n1.in‚0.078125* proc.memory.percent‚16.001558303833008 proc.cpu.threads‚17 network.sent‚ 2.3435686e+07 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚30345.7373046875 disk.sda2.in‚0 network.recv‚ 2.755885e+06 disk.drbd1716.in‚0) proc.memory.availableMB‚ 1058.93359375 disk.sda.in‚ 0.01171875^¯G2ž$- _wandbruntime‚7605  _runtime‚7605DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 320356781 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 139326642 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚608570911420928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚565278948082688 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚565278948082688 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚550660489273882 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚616996214774272 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8000604575258 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚557248128127488 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚557248128127488 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚81286766 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 139326644 _step‚786 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚557248110124544 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚565278273717248 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8030819955200 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 127096774 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚565278273717248 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 181050177 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚616996023015936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚608570935361024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚81286764 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚557248110124544 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 139326642 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8030163592704 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8425111595008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚542659901737984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8001259013014 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 139326644 _step‚786 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚557248110124544 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚565278273717248 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8030819955200 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 127096774 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚565278273717248 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 181050177 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚616996023015936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚608570935361024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚81286764 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚557248110124544 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 139326642 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8030163592704 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8425111595008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚542659901737984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8001259013014 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 320356781 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 139326642 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚608570911420928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚565278948082688 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚565278948082688 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚550660489273882 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚616996214774272 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8000604575258 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚557248128127488 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚557248128127488 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚81286766 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚787 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 139340658 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8001421010222 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚617058550766080 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚565335446183936 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚542714788829184 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 181068657 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 127109399 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚45814901 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚617058727831040 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚7605.884427067 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 139340658 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚565336120562688 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8030983020544 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 181048593 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 320389251 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 181068657 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚565336120562688 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 320389251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚557304481166336 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 139340660 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8425988607488 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚81294934 % _timestamp‚1.7884524904122536e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8030983020544 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚608632586098688 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚45814467 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚81294932 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚550716209839406 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 320409317 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0·ËÖ¦$ðM SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 127109399 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚45814901 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚617058727831040 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚7605.884427067 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 139340658 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚565336120562688 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8030983020544 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 181048593 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 320389251 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 181068657 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚565336120562688 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 320389251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚557304481166336 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 139340660 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8425988607488 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚81294934 % _timestamp‚1.7884524904122536e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8030983020544 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚608632586098688 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚45814467 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚81294932 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚550716209839406 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 L>DeviceStatsMonitor.on_train_batch_end/allocation.alÁ Èil.allocated‚ 320409317 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚787 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 139340658 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8001421010222 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚617058550766080 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚565335446183936 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚542714788829184 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 181068657 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8041439914666 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8071149398528 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 181956191 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 _runtime‚7643.233271717 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 127732515 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚46037349 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚560065762066944 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 140027442 % _timestamp‚1.7884525277614467e+09 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚620121702434816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚560065762066944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚568137585830912 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚560065780069888 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8071805761024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚568136911465472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8071805761024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚553447202618406 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8041439914666 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8071149398528 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 181956191 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 _runtime‚7643.233271717 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 127732515 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚46037349 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚560065762066944 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 140027442 % _timestamp‚1.7884525277614467e+09 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚620121702434816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚560065762066944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚568137585830912 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚560065780069888 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8071805761024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚568136911465472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8071805761024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚553447202618406 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 181974645 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 127744655 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 181994709 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚560122115105792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 322036169 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 322016103 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8469271590912 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8469101688832 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚560122115105792 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8072625202176 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 140041460 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚560122133108736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8071968826368 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚545460012396544 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚46041771 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚620184394572800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 322036169 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚568194083932160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚611715122981888 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 trainer/global_step‚9649 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8042256349630 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚ 7643.94115601Çcdiõ&¶$ïM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚568194083932160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚611715122981888 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 trainer/global_step‚9649 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8042256349630 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _runtime‚ 7643.94115601 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚791 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚46041323 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 181974645 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 127744655 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 181994709 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚560122115105792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 322036169 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 322016103 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8469271590912 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8469101688832 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚560122115105792 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8072625202176 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 140041460 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚560122133108736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8071968826368 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚545460012396544 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚46041771 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚620184394572800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 322036169 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0—·ª#Ú2·$- _wandbruntime‚7643  _runtime‚7643£)]X¸$Ò 0loss/FRt3/assignment_loss‚1.4579232163348441 /loss/FRt1/detection_loss‚2.7294195708301334 %loss/total_loss‚42.94092371017514 1loss/SRqqt4/assignment_loss‚1.261151386394364 2loss/SRqqt2/assignment_loss‚1.5260625158747037 1loss/FBt1/assignment_loss‚0.43629520371949254 /loss/FRt3/detection_loss‚1.7926857897033126 1loss/SRqqt3/detection_loss‚1.0838010245934129 trainer/global_step‚9649 _runtime‚7643.942453548 1loss/FBt2/assignment_loss‚0.30451008711295136 1loss/SRqqt1/detection_loss‚2.3993991365035376 1loss/SRqqt2/detection_loss‚1.7950616810056899 /loss/FBt3/detection_loss‚0.7371143703879435 epoch‚7 0loss/FBt4/assignment_loss‚0.5966406464576721 .loss/FRt4/detection_loss‚2.427771268920465 1loss/SRqqt4/detection_loss‚1.5557894257248426 /loss/FBt1/detection_loss‚2.3776497955505667 2loss/SRqqt1/assignment_loss‚2.1406110475460687 1loss/SRqqt3/assignment_loss‚1.006125485058874 /loss/FBt2/detection_loss‚2.2399055036631497 .loss/FBt4/detection_loss‚3.074008971452713 $ _timestamp‚1.788452528470955e+09 .loss/FRt1/assignment_loss‚4.52700415915913 /loss/FRt2/assignment_loss‚2.371592625975609 -loss/FRt2/detection_loss‚2.09866074770689 0loss/FRt4/assignment_loss‚2.8544839918613434 1loss/FBt3/assignment_loss‚0.14725605863743088 _step‚792V6bÚ]¹$× 0loss/FRt4/assignment_loss‚2.8544839918613434 1loss/FBt3/assignment_loss‚0.14725605863743088 _step‚792 0loss/FRt3/assignment_loss‚1.4579232163348441 /loss/FRt1/detection_loss‚2.7294195708301334 %loss/total_loss‚42.94092371017514 1loss/SRqqt4/assignment_loss‚1.261151386394364 2loss/SRqqt2/assignment_loss‚1.5260625158747037 1loss/FBt1/assignment_loss‚0.43629520371949254 /loss/FRt3/detection_loss‚1.7926857897033126 1loss/SRqqt3/detection_loss‚1.0838010245934129 trainer/global_step‚9649 _runtime‚7643.942453548 1loss/FBt2/assignment_loss‚0.30451008711295136 1loss/SRqqt1/detection_loss‚2.3993991365035376 1loss/SRqqt2/detection_loss‚1.7950616810056899 /loss/FBt3/detection_loss‚0.7371143703879435 epoch‚7 0loss/FBt4/assignment_loss‚0.5966406464576721 .loss/FRt4/detection_loss‚2.427771268920465 1loss/SRqqt4/detection_loss‚1.5557894257248426 /loss/FBt1/detection_loss‚2.3776497955505667 2loss/SRqqt1/assignment_loss‚2.1406110475460687 1loss/SRqqt3/assignment_loss‚1.006125485058874 /loss/FBt2/detection_loss‚2.2399055036631497 .loss/FBt4/detection_loss‚3.074008971452713 $ _timestamp‚1.788452528470955e+09 .loss/FRt1/assignment_loss‚4.52700415915913 /loss/FRt2/assignment_loss‚2.371592625975609 -loss/FRt2/detection_loss‚2.09866074770689˜–ØB!žº$‚(:“ ¶½æÔ·íü gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.932' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.986399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0º­l_o»$‚(:ä ¶½æÔ öŽ€ disk.sda2.out‚0) proc.memory.availableMB‚ 1059.08203125) disk./.usagePercent‚20.34505238200879 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚30486.85546875 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 proc.cpu.threads‚17 network.sent‚ 2.3556703e+07 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-0.in‚0.078125 network.recv‚ 2.771417e+06# disk.nvme0n1.out‚30486.85546875 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 294.08203125 disk.rbd0.out‚0 disk.dm-0.out‚30486.85546875 disk.sda.out‚ 294.08203125$ memory_percent‚93.53587627410889 disk.dm-1.out‚ 294.08203125 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 44.1474609375 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0 disk.drbd2080.in‚0* proc.memory.percent‚16.001558303833008 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125" proc.memory.rssMB‚ 2621.6953125 cpu‚25.325010366920267$ disk./.usageGB‚303.1559829711914år‘¼$‚(:’ ŽæÔñïšý gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚123.266' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.653200000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0OÜT8k½$‚(:à ŽæÔ­¢ÿ) proc.memory.percent‚16.14522933959961 network.sent‚ 2.3664378e+07) disk./.usagePercent‚20.34769514778475' disk.nvme0c0n1.out‚30533.5166015625 disk.sda.in‚ 0.01171875 disk.drbd2265.out‚0 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2265.in‚0' proc.memory.availableMB‚ 1035.078125 disk.sda.out‚ 294.4140625# memory_percent‚93.6823844909668! proc.memory.rssMB‚ 2645.234375 disk.dm-1.out‚ 294.4140625 network.recv‚ 2.7767e+06 disk.sda3.out‚ 294.4140625 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125" disk.dm-0.out‚30533.5166015625 disk.drbd1716.out‚0 disk.rbd0.out‚0 disk.sda1.out‚0 cpu‚25.32730676381135 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0% disk./.usageGB‚303.19536209106445 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.out‚0 disk.rbd1.out‚ 44.1474609375 disk.rbd1.in‚0% disk.nvme0n1.out‚30533.5166015625 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.sda2.in‚0¼y¡¾$‚(:’ Ô½æÔ͉—ý gpu.0.gpu‚7.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.969' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.593799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0$q„h¿$‚(:Ý Ô½æÔ“‘“ÿ* disk./.usagePercent‚20.348719945944318 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.sda1.out‚0! proc.memory.rssMB‚ 2645.234375 disk.drbd2266.out‚0 network.recv‚ 2.781432e+06 disk.drbd2265.in‚0( disk.nvme0c0n1.out‚30597.44580078125 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 44.1796875 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.rbd0.in‚0 disk.rbd0.out‚0# disk.dm-0.out‚30597.44580078125( proc.memory.availableMB‚ 1035.2578125 disk.dm-1.out‚295.0625 disk.sda3.in‚ 0.01171875) proc.memory.percent‚16.14522933959961 disk.rbd1.in‚0 disk.dm-0.in‚0.078125 network.sent‚ 2.3670858e+07& disk.nvme0n1.out‚30597.44580078125 disk.drbd1716.in‚0 disk.sda3.out‚295.0625$ memory_percent‚93.68128776550293 cpu‚25.336756022956216% disk./.usageGB‚303.21063232421875 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda.out‚295.0625 disk.sda1.in‚0 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875NÌ¿ À$‚(:’ ã½æÔ×çŽý gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚122.865' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.572999999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ê Öå]Á$‚(:Ò ã½æÔø¡€ disk.drbd2266.out‚0# disk.dm-0.out‚30641.83251953125 disk.sda.in‚ 0.01171875 disk.sda.out‚295.625 disk.sda1.out‚0$ disk./.usageGB‚303.2184181213379 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.dm-0.in‚0.078125 disk.rbd1.in‚0 disk.rbd1.out‚ 44.2890625! proc.memory.rssMB‚ 2645.234375& disk.nvme0n1.out‚30641.83251953125 disk.drbd1716.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 cpu‚25.345480596147617 disk.sda2.in‚0# proc.memory.availableMB‚1035.25( disk./.usagePercent‚20.3492424573217 disk.dm-1.out‚295.625 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125$ memory_percent‚93.68133544921875 network.recv‚ 2.784558e+06) proc.memory.percent‚16.14522933959961 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚30641.83251953125 disk.sda3.out‚295.625 disk.rbd0.out‚0 network.sent‚ 2.3676003e+07 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0솕pÂ$‚(:å ò½æÔõå”ÿ network.recv‚ 2.789423e+06 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0$ disk./.usageGB‚303.2261962890625 disk.sda.in‚ 0.01171875 disk.dm-1.out‚ 296.3515625 disk.drbd2265.in‚0 disk.sda3.out‚ 296.3515625 disk.dm-0.in‚0.078125$ memory_percent‚93.68281364440918! proc.memory.rssMB‚ 2645.234375 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0# disk.dm-0.out‚30689.81591796875 disk.drbd2266.out‚0* disk./.usagePercent‚20.349764456684017 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0) proc.memory.percent‚16.14522933959961 proc.cpu.threads‚17& disk.nvme0n1.out‚30689.81591796875 network.sent‚ 2.3682635e+07( proc.memory.availableMB‚ 1035.0078125 disk.sda.out‚ 296.3515625 disk.sda1.in‚0( disk.nvme0c0n1.out‚30689.81591796875 disk.rbd0.in‚0 disk.rbd0.out‚0 cpu‚25.354380126281473 disk.rbd1.out‚ 44.3515625 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875m·^’Ã$‚(:‡ ò½æÔõ¸© gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚186.233' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.2466 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0®/2Ä$- _wandbruntime‚7711  _runtime‚7711C!¢Å$ÆT D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15979 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚563166571374592 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15979 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚556475498291686 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.freed‚0 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 8078217847270 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚ 140879306 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 8108562162688 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 182758607 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚82201726 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 RDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚ 140879304 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 trainer/global_step‚9671 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚571274622704128 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 113246720 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚571275133537280 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚155 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 FDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 $ _timestamp‚1.788452595749632e+09 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 8108069332480 OADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚ 323621934 RDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚ 182742630 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 VHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚ 182758607 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 _runtime‚ 7711.22168209 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 323621934 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚623433476784640 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 _step‚793 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492830208 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚548397297483776 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚571274622704128 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚623433613971456 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 8078708761662 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚563166553371648 SEDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚ 323637913 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 182742630 WIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚ 128560965 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 339635712 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 8506263819264 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚556476006245438 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 323637913 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚82201724 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚46359084‰ZšQ*Æ$ËT [GDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated‚571275133537280 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 8108562162688 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 8506150572544 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 $ _timestamp‚1.788452595749632e+09 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 8108069332480 OADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚ 323621934 RDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚ 182742630 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 VHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚ 182758607 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 _runtime‚ 7711.22168209 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 323621934 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚623433476784640 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 _step‚793 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492830208 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚548397297483776 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚571274622704128 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚623433613971456 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 8078708761662 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚563166553371648 SEDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚ 323637913 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 182742630 WIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚ 128560965 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 339635712 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 8506263819264 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚556476006245438 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 323637913 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚82201724 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚46359084 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15979 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚563166571374592 MCDeviceStatsMonitor.on_validation_batch_start/allocation.all.current‚15979 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚556475498291686 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.freed‚0 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 8078217847270 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ^PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚ 140879306 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 8108562162688 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 182758607 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚82201726 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 RDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚ 140879304 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 trainer/global_step‚9671 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚571274622704128 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 113246720 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚571275133537280 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚155 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 FDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 ICDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚563203401403392 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 OEDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.peak‚24128 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.peak‚889 MGDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚ 128572039 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 8108434459136 aODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.allocated‚ 8108927289344 XJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.peak‚ 863815680 M?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚ 323640042 ]IDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.freed‚623471717644800 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 XJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚ 140890112 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 339635712 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 RCDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚563203401403392 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚563203383400448 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚82208228 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414 PBDeviceStatsMonitor.on_validation_batch_end/active.large_pool.freed‚ 140890110 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚46363809 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 TFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚ 140890112 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 323656021 JDDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.current‚ 510833152 ^LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚ 8108927289344 TFDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.freed‚ 182749932 LFDeviceStatsMonitor.on_validation_batch_end/oversize_segments.allocated‚0 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 % _timestamp‚1.7884525967154574e+09 _step‚794 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24401 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated‚571312328692736 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 ICDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚563203401403392 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 OEDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.peak‚24128 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 QIDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.peak‚889 MGDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚ 128572039 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 8108434459136 aODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.allocated‚ 8108927289344 XJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.peak‚ 863815680 M?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚ 323640042 ]IDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.freed‚623471717644800 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 XJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚ 140890112 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 339635712 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 RCDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚563203401403392 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚563203383400448 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚82208228 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414š´·»¢žÊ$‚(:“ ¾æÔψŠþ gpu.0.gpu‚29.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.969' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚24.593799999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0]†wmUË$‚(:Ê ¾æÔûÀ†ÿ network.sent‚ 2.3687977e+07* disk./.usagePercent‚20.350118003088802 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.dm-1.out‚297 disk.drbd2265.in‚0 network.recv‚ 2.792828e+06 disk.drbd2266.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0! proc.memory.rssMB‚ 2645.234375$ disk./.usageGB‚303.2314643859863 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125 disk.drbd1716.in‚0# memory_percent‚93.6826229095459 disk.drbd2080.in‚0 disk.sda3.out‚297 disk.rbd0.out‚0 disk.sda2.in‚0& disk.nvme0n1.out‚30736.95263671875 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.sda1.out‚0 disk.rbd1.out‚44.53125# disk.dm-0.out‚30736.95263671875 disk.sda.in‚ 0.01171875 disk.sda.out‚297 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0( disk.nvme0c0n1.out‚30736.95263671875( proc.memory.availableMB‚ 1035.0390625) proc.memory.percent‚16.14522933959961 cpu‚25.363496485093403{=M—2Ì$- _wandbruntime‚7725  _runtime‚7725¥VB³bÍ$Ä¥ >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4633333086967468 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9633333683013916 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5861819982528687 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6416769623756409 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9859274625778198 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1409674882888794 * Purity/1FRt3SRqqt1FBt/FRt_purity‚0.375 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.35185182094573975 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9821428656578064 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5922619104385376 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.18328788876533508 6 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.270537406206131 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_4_of_5‚1 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.14823481440544128 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.12866899371147156 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.12444444745779037 ,"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.375 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9881592988967896 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.2666667103767395 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.45711052417755127 6 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.992674708366394 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 jet/accuracy_6_of_6‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.06028369069099426 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5140703916549683 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.27931031584739685 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.6042646169662476 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8888888955116272 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9803187847137451 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.19207903742790222 ,"Purity/1FRt0SRqqt3FBt/event_purity‚0.125 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 9"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.4621809124946594 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 *jet/accuracy_1_of_1‚0.5163705348968506 6validation_average_jet_accuracy‚0.4520212411880493 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5376321077346802 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_4‚1 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 particle/accuracy_5_of_7‚1 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.10052070766687393 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.13651394844055176 8"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.566691517829895 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 +"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.25 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9923497438430786 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.42344963550567627 9"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.4518018066883087 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.4703703820705414 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4305556118488312 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5680555701255798 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4839080572128296 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 (particle/accuracy‚0.8501518368721008 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.15287356078624725 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2271825224161148 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.19067135453224182 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0972222313284874 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 8 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.40963485836982727 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.34375 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 ,jet/accuracy_4_of_5‚0.001792114693671465 particle/accuracy_2_of_5‚1 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.20293211936950684 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.011290322057902813 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9597700834274292 -jet/accuracy_4_of_4‚0.0018155410652980208 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.17424242198467255 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5460623502731323 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5658104419708252 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9896097779273987 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4703703820705414 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5711309909820557 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_6‚1 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.21403513848781586 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.10238095372915268 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚1 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 trainer/global_step‚9671 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.20565931499004364 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 8 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.39772728085517883 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 jet/accuracy_7_of_7‚0 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09355352073907852 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5595042705535889 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1795647144317627 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 +"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.25 particle/accuracy_5_of_6‚1 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9744047522544861 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.10580119490623474 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 +jet/accuracy_3_of_3‚0.02110419236123562 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 jet/accuracy_3_of_7‚0 jet/accuracy_5_of_7‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5007117986679077 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.1111111119389534 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4493163824081421 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5262449383735657 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9946674704551697 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 jet/accuracy_1_of_6‚1 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.10555118322372437 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17761091887950897 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5452442765235901 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.01320754736661911 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.10097453743219376 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5096275210380554 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_5‚1 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.24125395715236664 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9936118125915527 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.0432753823697567 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.17274776101112366 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 jet/accuracy_1_of_7‚1 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5467271208763123 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.4686827063560486 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.017080215737223625 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 6 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.979221522808075 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 7 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.1292397528886795 7 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.1331985741853714 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0972222313284874 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5963277220726013 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5500448942184448 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.3333333134651184 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9840376973152161 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24534527957439423 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.6315789222717285 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 particle/accuracy_1_of_1‚1 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.6176470518112183 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.08176238089799881 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3531813323497772 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_2_of_7‚1 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6812238097190857 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4270833134651184 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.10731694102287292 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.13122332096099854 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.6007799506187439 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9628472924232483 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.41995614767074585 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9877192974090576 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7121211886405945 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5357142686843872 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9290780425071716 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.6466667056083679 8"Purity/*FRt*SRqqt1FBt/event_purity‚0.402985543012619 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2596075236797333 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9444443583488464 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9830172657966614 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 *jet/accuracy_1_of_5‚0.9743728041648865 particle/accuracy_3_of_5‚1 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.29409152269363403 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 8"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.587128758430481 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.012001381255686283 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5606921315193176 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.4259259104728699 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.3460807204246521 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.05263157933950424 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 particle/accuracy_2_of_6‚1 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0019919080659747124 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.4554040729999542 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9935437440872192 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.5278024077415466 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.998068630695343 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.14174038171768188 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9893779158592224 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5468600988388062 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5461942553520203 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3263888955116272 particle/accuracy_6_of_6‚1 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9952660202980042 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.005000146571546793 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 particle/accuracy_7_of_7‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17955763638019562 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.09249208122491837 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.14772461354732513 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.1857023984193802 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 _step‚795 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5597834587097168 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.03323632478713989 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.016144437715411186 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5650415420532227 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 & Purity/*FRt0SRqqt4FBt/FRt_purity‚1 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7424241900444031 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 particle/accuracy_2_of_2‚1 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 7 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.0833333358168602 jet/accuracy_5_of_5‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.6052631735801697 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08803483098745346 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 *jet/accuracy_2_of_5‚0.5202251672744751 particle/accuracy_1_of_2‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0793650820851326 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.34167569875717163 5 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.98872971534729 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.6005346179008484 7 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.1301795244216919 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.6190889477729797 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_4_of_6‚1 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 + Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0625 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 -"Purity/0FRt0SRqqt3FBt/event_purity‚0.0625 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 * Purity/4FRt*SRqqt0FBt/FRt_purity‚0.125 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.20525838434696198 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.007275132462382317 * Purity/1FRt0SRqqt3FBt/FRt_purity‚0.625 7 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.1849358081817627 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 ) Purity/4FRt1SRqqt*FBt/FRt_purity‚0.25 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 (particle/f_score‚0.40464720129966736 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3214285671710968 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9660778641700745 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.25317925214767456 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.04104308411478996 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 jet/accuracy_4_of_6‚0 7 Purity/*FRt1Sm¤FÛ0Rqqt1FBt/FRt_purity‚0.3094708323478699 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.47385621070861816 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_7‚1 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ) Purity/4FRt1SRqqt0FBt/FRt_purity‚0.25 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_2‚0.7708497047424316 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.015630703419446945 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.37931543588638306 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 % _timestamp‚1.7884526099224105e+09 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 particle/accuracy_5_of_5‚1 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.26539209485054016 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5790350437164307 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 *jet/accuracy_1_of_4‚0.9220743775367737 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.21021075546741486 7 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.1372392475605011 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.16886933147907257 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5519214272499084 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 8"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.602861225605011 6 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.385416716337204 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1795647144317627 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 +jet/accuracy_2_of_3‚0.24817916750907898 jet/accuracy_6_of_7‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.19740766286849976 *jet/accuracy_2_of_4‚0.3653023838996887 7 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.8571428656578064 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 6 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.983968198299408 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5745617151260376 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5972798466682434 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 7"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.54844731092453 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.2380952537059784 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.5027925372123718 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9479167461395264 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.29676809906959534 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 jet/accuracy_5_of_6‚0 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 )loss/val_total_loss‚45.13168086652367 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 jet/accuracy_2_of_7‚0.5 particle/accuracy_1_of_7‚1 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5796708464622498 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 particle/accuracy_2_of_3‚1 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.4197531044483185 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4969984292984009 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8843057155609131 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.3499999940395355 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5949074625968933 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.4046626687049866 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.6388888955116272 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 particle/accuracy_4_of_4‚1 particle/accuracy_3_of_6‚1 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3529677093029022 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5649664998054504 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.015804599970579147 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4333333373069763 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9892290234565735 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5875358581542969 7"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.65123450756073 _runtime‚7725.396311001 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6522988677024841 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.7708333134651184 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9904961585998535 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.9210527539253235 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9823634028434753 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9929894208908081 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3263888955116272 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.2494981437921524 +jet/accuracy_3_of_5‚0.09045058488845825 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 jet/accuracy_4_of_7‚0 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5649664998054504 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_7‚1 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 * Purity/4FRt*SRqqt*FBt/FRt_purity‚0.125 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.027738893404603004 7 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.1510239839553833 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 particle/accuracy_3_of_4‚1 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 +jet/accuracy_2_of_6‚0.33974361419677734 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.7708333134651184 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.15191884338855743 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 8"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.571148693561554 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.14218711853027344 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4697318375110626 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 +jet/accuracy_3_of_4‚0.05016452074050903 particle/accuracy_3_of_3‚1 + Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0625 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.32172614336013794 +particle/sensitivity‚0.3263911008834839 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5515738725662231 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.11015326529741287 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.2528626620769501 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9966205954551697 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.38292941451072693 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9901832342147827 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5596612095832825 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5672774910926819 7 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.1633838266134262 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6365005373954773 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 9"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.4709150195121765 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0028860028833150864 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 +particle/specificity‚0.9470549821853638 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 *jet/accuracy_3_of_6‚0.1538461595773697 7 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.2725146412849426 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 epoch‚7 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13130153715610504 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5240063071250916 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5282727479934692 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 +jet/accuracy_2_of_2‚0.10740234702825546 particle/accuracy_1_of_3‚1 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8162482380867004 8"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.569725751876831 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.1318623274564743 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 particle/accuracy_6_of_7‚1 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 9"Purity/*FRt2SRqqt*FBt/event_purity‚0.0366302952170372 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 ,"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.375 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.40141093730926514^œˆÚOÎ$É¥ =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9923497438430786 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.42344963550567627 9"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.4518018066883087 9"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.4703703820705414 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 7 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.4305556118488312 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5680555701255798 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4839080572128296 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 (particle/accuracy‚0.8501518368721008 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.15287356078624725 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2271825224161148 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 8 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.19067135453224182 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0972222313284874 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 8 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.40963485836982727 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.34375 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 ,jet/accuracy_4_of_5‚0.001792114693671465 particle/accuracy_2_of_5‚1 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.20293211936950684 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.011290322057902813 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9597700834274292 -jet/accuracy_4_of_4‚0.0018155410652980208 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.17424242198467255 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5460623502731323 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5658104419708252 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9896097779273987 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4703703820705414 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5711309909820557 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_6‚1 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.21403513848781586 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.10238095372915268 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚1 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 trainer/global_step‚9671 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.20565931499004364 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 8 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.39772728085517883 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 jet/accuracy_7_of_7‚0 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09355352073907852 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5595042705535889 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1795647144317627 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 +"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.25 particle/accuracy_5_of_6‚1 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9744047522544861 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.10580119490623474 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 +jet/accuracy_3_of_3‚0.02110419236123562 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 jet/accuracy_3_of_7‚0 jet/accuracy_5_of_7‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5007117986679077 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.1111111119389534 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4493163824081421 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5262449383735657 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9946674704551697 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 jet/accuracy_1_of_6‚1 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.10555118322372437 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17761091887950897 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5452442765235901 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.01320754736661911 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.10097453743219376 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5096275210380554 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_5‚1 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.24125395715236664 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9936118125915527 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.0432753823697567 8 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.17274776101112366 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 jet/accuracy_1_of_7‚1 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5467271208763123 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.4686827063560486 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.017080215737223625 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 6 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.979221522808075 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 7 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.1292397528886795 7 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.1331985741853714 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0972222313284874 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5963277220726013 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5500448942184448 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.3333333134651184 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9840376973152161 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24534527957439423 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.6315789222717285 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 particle/accuracy_1_of_1‚1 -"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.5625 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.6176470518112183 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.08176238089799881 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3531813323497772 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_2_of_7‚1 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6812238097190857 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4270833134651184 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.10731694102287292 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.13122332096099854 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.6007799506187439 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9628472924232483 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.41995614767074585 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 7 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.9877192974090576 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7121211886405945 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5357142686843872 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9290780425071716 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.6466667056083679 8"Purity/*FRt*SRqqt1FBt/event_purity‚0.402985543012619 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2596075236797333 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9444443583488464 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9830172657966614 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 *jet/accuracy_1_of_5‚0.9743728041648865 particle/accuracy_3_of_5‚1 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.29409152269363403 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 8"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.587128758430481 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.012001381255686283 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5606921315193176 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.4259259104728699 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.3460807204246521 :"Purity/0FRt*SRqqt3FBt/event_purity‚0.05263157933950424 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 particle/accuracy_2_of_6‚1 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0019919080659747124 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.4554040729999542 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9935437440872192 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.5278024077415466 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.998068630695343 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.14174038171768188 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9893779158592224 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5468600988388062 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5461942553520203 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9922481179237366 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3263888955116272 particle/accuracy_6_of_6‚1 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9952660202980042 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.005000146571546793 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 particle/accuracy_7_of_7‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17955763638019562 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.09249208122491837 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9979166388511658 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.14772461354732513 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.1857023984193802 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 _step‚795 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5597834587097168 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.03323632478713989 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.016144437715411186 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5650415420532227 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 & Purity/*FRt0SRqqt4FBt/FRt_purity‚1 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7424241900444031 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 particle/accuracy_2_of_2‚1 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 7 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.0833333358168602 jet/accuracy_5_of_5‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.6052631735801697 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08803483098745346 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 *jet/accuracy_2_of_5‚0.5202251672744751 particle/accuracy_1_of_2‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0793650820851326 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.34167569875717163 5 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.98872971534729 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.6005346179008484 7 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.1301795244216919 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.6190889477729797 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_4_of_6‚1 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 + Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0625 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 -"Purity/0FRt0SRqqt3FBt/event_purity‚0.0625 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 * Purity/4FRt*SRqqt0FBt/FRt_purity‚0.125 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.20525838434696198 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.007275132462382317 * Purity/1FRt0SRqqt3FBt/FRt_purity‚0.625 7 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.1849358081817627 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 ) Purity/4FRt1SRqqt*FBt/FRt_purity‚0.25 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 (particle/f_score‚0.40464720129966736 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3214285671710968 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9660778641700745 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.25317925214767456 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.04104308411478996 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 jet/accuracy_4_of_6‚0 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.3094708323478699 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.47385621070861816 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_7‚1 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 ,&Purity/1FRt2SRqqt3FBt/event_proportiî1¯¯öBon‚0 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ) Purity/4FRt1SRqqt0FBt/FRt_purity‚0.25 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_2‚0.7708497047424316 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.015630703419446945 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.37931543588638306 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 % _timestamp‚1.7884526099224105e+09 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 particle/accuracy_5_of_5‚1 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.26539209485054016 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5790350437164307 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 *jet/accuracy_1_of_4‚0.9220743775367737 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.21021075546741486 7 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.1372392475605011 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.16886933147907257 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5519214272499084 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 8"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.602861225605011 6 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.385416716337204 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1795647144317627 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 +jet/accuracy_2_of_3‚0.24817916750907898 jet/accuracy_6_of_7‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.19740766286849976 *jet/accuracy_2_of_4‚0.3653023838996887 7 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.8571428656578064 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 6 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.983968198299408 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5745617151260376 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5972798466682434 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 7"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.54844731092453 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.2380952537059784 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.5027925372123718 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9479167461395264 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.29676809906959534 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 jet/accuracy_5_of_6‚0 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 )loss/val_total_loss‚45.13168086652367 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 jet/accuracy_2_of_7‚0.5 particle/accuracy_1_of_7‚1 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5796708464622498 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 particle/accuracy_2_of_3‚1 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.4197531044483185 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.4969984292984009 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8843057155609131 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.3499999940395355 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.5949074625968933 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.4046626687049866 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.6388888955116272 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 particle/accuracy_4_of_4‚1 particle/accuracy_3_of_6‚1 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3529677093029022 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5649664998054504 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.015804599970579147 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4333333373069763 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9892290234565735 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5875358581542969 7"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.65123450756073 _runtime‚7725.396311001 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6522988677024841 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.7708333134651184 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9904961585998535 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.9210527539253235 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9823634028434753 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9929894208908081 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3263888955116272 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.2494981437921524 +jet/accuracy_3_of_5‚0.09045058488845825 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 jet/accuracy_4_of_7‚0 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5649664998054504 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_7‚1 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9886363744735718 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 * Purity/4FRt*SRqqt*FBt/FRt_purity‚0.125 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.027738893404603004 7 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.1510239839553833 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 particle/accuracy_3_of_4‚1 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 +jet/accuracy_2_of_6‚0.33974361419677734 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.7708333134651184 ( Purity/1FRt1SRqqt3FBt/FRt_purity‚0.5 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.15191884338855743 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 8"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.571148693561554 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.14218711853027344 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4697318375110626 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 +jet/accuracy_3_of_4‚0.05016452074050903 particle/accuracy_3_of_3‚1 + Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0625 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 5 Purity/0FRt2SRqqt1FBt/FBt_purity‚0.99622642993927 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.32172614336013794 +particle/sensitivity‚0.3263911008834839 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5515738725662231 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.11015326529741287 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.2528626620769501 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9966205954551697 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.38292941451072693 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9901832342147827 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5596612095832825 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5672774910926819 7 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.1633838266134262 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6365005373954773 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 9"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.4709150195121765 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 <"Purity/1FRt2SRqqt0FBt/event_purity‚0.0028860028833150864 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 +particle/specificity‚0.9470549821853638 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 *jet/accuracy_3_of_6‚0.1538461595773697 7 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.2725146412849426 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 epoch‚7 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13130153715610504 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5240063071250916 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5282727479934692 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 +jet/accuracy_2_of_2‚0.10740234702825546 particle/accuracy_1_of_3‚1 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8162482380867004 8"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.569725751876831 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.1318623274564743 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 particle/accuracy_6_of_7‚1 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 9"Purity/*FRt2SRqqt*FBt/event_purity‚0.0366302952170372 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 ,"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.375 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.40141093730926514 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 9"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.4633333086967468 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9633333683013916 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5861819982528687 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6416769623756409 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9859274625778198 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1409674882888794 * Purity/1FRt3SRqqt1FBt/FRt_purity‚0.375 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.35185182094573975 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9821428656578064 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5922619104385376 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 :"Purity/*FRt*SRqqt2FBt/event_purity‚0.18328788876533508 6 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.270537406206131 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_4_of_5‚1 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.14823481440544128 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.12866899371147156 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.12444444745779037 ,"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.375 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9881592988967896 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.2666667103767395 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.45711052417755127 6 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.992674708366394 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 jet/accuracy_6_of_6‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.06028369069099426 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5140703916549683 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.27931031584739685 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.6042646169662476 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8888888955116272 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9803187847137451 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.19207903742790222 ,"Purity/1FRt0SRqqt3FBt/event_purity‚0.125 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 9"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.4621809124946594 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 *jet/accuracy_1_of_1‚0.5163705348968506 6validation_average_jet_accuracy‚0.4520212411880493 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5376321077346802 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_4‚1 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 particle/accuracy_5_of_7‚1 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.10052070766687393 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.13651394844055176 8"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.566691517829895 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ( Purity/*FRt1SRqqt3FBt/FRt_purity‚0.5 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 +"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.25 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1›f_¸žÏ$‚(:“ ¾æÔšÀòý gpu.0.gpu‚40.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚178.985' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.797000000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0(ÔMycÐ$‚(:Ø ¾æÔÉ®ž€ disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.rbd0.out‚0 disk.dm-0.out‚ 31290.078125 disk.sda2.in‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 297.62890625 cpu‚25.362046579593407 disk.dm-1.in‚ 0.01171875# disk.nvme0c0n1.out‚ 31290.078125 disk.sda3.out‚ 297.62890625 disk.rbd0.in‚0 disk.sda3.in‚ 0.01171875 disk.rbd1.in‚0 disk.rbd1.out‚44.53125 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125! proc.memory.rssMB‚ 2645.234375 network.recv‚ 2.799552e+06) proc.memory.percent‚16.14522933959961 disk.drbd1716.out‚0) disk./.usagePercent‚20.35067328343057 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.sda1.out‚0 network.sent‚ 2.3796638e+07 disk.drbd2266.out‚0$ memory_percent‚91.29116535186768) proc.memory.availableMB‚ 1426.85546875 disk.dm-0.in‚0.078125% disk./.usageGB‚303.23973846435547 disk.sda2.out‚0 disk.dm-1.out‚ 297.62890625 disk.drbd2080.in‚0 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125! disk.nvme0n1.out‚ 31290.078125‹!ø 2Ñ$- _wandbruntime‚7750  _runtime‚7750HôöAÈÒ$ _step‚796 ' lr-AdamW/pg1‚1.4503486317096319e-05 ' lr-AdamW/pg2‚1.4503486317096319e-05 trainer/global_step‚9699 $ _timestamp‚1.788452634719869e+09 _runtime‚7750.191438952ø6YÍÓ$Ç ' lr-AdamW/pg2‚1.4503486317096319e-05 trainer/global_step‚9699 $ _timestamp‚1.788452634719869e+09 _runtime‚7750.191438952 _step‚796 ' lr-AdamW/pg1‚1.4503486317096319e-05œ¸SÓ2Ô$- _wandbruntime‚7750  _runtime‚7750•÷÷ç'Õ$áO WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8135155174400 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚46543691 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚617122156398080 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 183344206 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 141409020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚550373821641728 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚550373804602368 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚9699 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚565203705823744 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8135811536896 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 183344206 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 _step‚797 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚558479675833640 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8105199754156 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 141409022 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚625656947815936 _runtime‚7750.192346826 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚573338860998144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8135155174400 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8534767477760 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚82513318 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 324773266 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚573339535363584 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚449 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚573338860998144 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚46543242 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872P⿯ê Ö$æO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8135811536896 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 183344206 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 _step‚797 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚558479675833640 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8105199754156 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 141409022 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚625656947815936 _runtime‚7750.192346826 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚573338860998144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8135155174400 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8534767477760 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚82513318 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 324773266 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚573339535363584 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚449 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚573338860998144 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚46543242 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8135155174400 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚46543691 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚617122156398080 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 183344206 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 141409020 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚550373821641728 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚550373804602368 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 trainer/global_step‚9699 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚451 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚565203705823744cÏÊ™2×$- _wandbruntime‚7750  _runtime‚7750~(‚ñ&Ø$ëM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚573396033464832 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 129069567 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _step‚798 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 183382724 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚565260058862592 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚565260058862592 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 183362660 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚46547630 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8136630978048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚558534724922048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚565260076865536 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚617183807135744 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 141423038 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚573396707843584 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 324805762 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚550428725772288 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 141423038 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8535628664320 % _timestamp‚1.7884526354375153e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8106670627084 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 324785696 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 183362660 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8106016189120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 324805762 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚46548081 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 183382724 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚617183831075840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 129069114 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 trainer/global_step‚9699 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 141423036 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚82521486 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚7750.909813591 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚550428708732928“è@Îö&Ù$ðM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8106670627084 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 324785696 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 183362660 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8106016189120 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 324805762 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚46548081 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 183382724 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚617183831075840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 129069114 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 trainer/global_step‚9699 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 141423036 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚82521486 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚7750.909813591 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚550428708732928 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚573396033464832 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 129069567 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _step‚798 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 183382724 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚565260058862592 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚565260058862592 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 183362660 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚46547630 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8136630978048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚558534724922048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚565260076865536 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚617183807135744 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 141423038 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚573396707843584 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 324805762 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚550428725772288 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 141423038 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8535628664320 % _timestamp‚1.7884526354375153e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0žá;†ç2Ú$- _wandbruntime‚7750  _runtime‚7750‚v¹Ì[Û$Õ /loss/FBt2/assignment_loss‚0.333725624686726 2loss/SRqqt3/assignment_loss‚1.1613391888520075 0loss/FBt3/assignment_loss‚0.1736305254077972 /loss/FRt1/detection_loss‚2.8654768062489375 1loss/SRqqt2/detection_loss‚1.6202717858056228 .loss/FBt2/detection_loss‚2.221212959289551 .loss/FRt2/detection_loss‚2.305369982761996 0loss/SRqqt1/detection_loss‚2.165422575040297 /loss/FBt1/detection_loss‚2.6674825378826688 0loss/FRt3/assignment_loss‚2.0968479235121547 0loss/FRt4/assignment_loss‚2.8057254856708003 2loss/SRqqt2/assignment_loss‚1.6696953812821045 2loss/SRqqt4/assignment_loss‚1.1683597810435713 /loss/FRt3/detection_loss‚1.8189710253413687 0loss/SRqqt4/detection_loss‚1.669457521354943 trainer/global_step‚9699 0loss/FRt1/assignment_loss‚3.9423744721072063 1loss/FBt4/assignment_loss‚0.09845628486755231 _runtime‚7750.911121871 2loss/SRqqt1/assignment_loss‚1.6415228351499096 _step‚799 % _timestamp‚1.7884526354395602e+09 0loss/FRt2/assignment_loss‚3.1900727975936163 1loss/FBt1/assignment_loss‚0.35136587092469945 .loss/FRt4/detection_loss‚2.153029711223116 /loss/FBt3/detection_loss‚0.5465434512441766 &loss/total_loss‚43.062976094254395 0loss/SRqqt3/detection_loss‚1.153278714371106 /loss/FBt4/detection_loss‚3.2433428525924684 epoch‚8÷g•A`Ü$Ú % _timestamp‚1.7884526354395602e+09 0loss/FRt2/assignment_loss‚3.1900727975936163 1loss/FBt1/assignment_loss‚0.35136587092469945 .loss/FRt4/detection_loss‚2.153029711223116 /loss/FBt3/detection_loss‚0.5465434512441766 &loss/total_loss‚43.062976094254395 0loss/SRqqt3/detection_loss‚1.153278714371106 /loss/FBt4/detection_loss‚3.2433428525924684 epoch‚8 /loss/FBt2/assignment_loss‚0.333725624686726 2loss/SRqqt3/assignment_loss‚1.1613391888520075 0loss/FBt3/assignment_loss‚0.1736305254077972 /loss/FRt1/detection_loss‚2.8654768062489375 1loss/SRqqt2/detection_loss‚1.6202717858056228 .loss/FBt2/detection_loss‚2.221212959289551 .loss/FRt2/detection_loss‚2.305369982761996 0loss/SRqqt1/detection_loss‚2.165422575040297 /loss/FBt1/detection_loss‚2.6674825378826688 0loss/FRt3/assignment_loss‚2.0968479235121547 0loss/FRt4/assignment_loss‚2.8057254856708003 2loss/SRqqt2/assignment_loss‚1.6696953812821045 2loss/SRqqt4/assignment_loss‚1.1683597810435713 /loss/FRt3/detection_loss‚1.8189710253413687 0loss/SRqqt4/detection_loss‚1.669457521354943 trainer/global_step‚9699 0loss/FRt1/assignment_loss‚3.9423744721072063 1loss/FBt4/assignment_loss‚0.09845628486755231 _runtime‚7750.911121871 2loss/SRqqt1/assignment_loss‚1.6415228351499096 _step‚799Ÿ\Û“ žÝ$‚(:“ Ÿ¾æÔÒËøü gpu.0.gpu‚43.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚145.161' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.032200000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ž5ÈÐhÞ$‚(:Ý Ÿ¾æԦъ€ disk.rbd0.in‚0 disk.rbd0.out‚0# disk.dm-0.out‚31371.59423828125& disk.nvme0n1.out‚31371.59423828125 cpu‚25.36141421875271 disk.drbd2265.in‚0 disk.sda.in‚ 0.01171875) proc.memory.percent‚16.14522933959961 disk.drbd2266.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875* disk./.usagePercent‚20.351043214317546 disk.sda1.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125$ memory_percent‚91.29142761230469 disk.drbd2080.out‚0 network.sent‚ 2.391963e+07 network.recv‚ 2.803885e+06 disk.drbd2266.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.out‚ 298.1328125 proc.cpu.threads‚17$ disk./.usageGB‚303.2452507019043 disk.sda.out‚ 298.1328125 disk.sda2.out‚0 disk.rbd1.out‚44.6875 disk.nvme0n1.in‚0.078125 disk.sda3.out‚ 298.1328125! proc.memory.rssMB‚ 2645.234375 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0% proc.memory.availableMB‚ 1426.8125 disk.dm-0.in‚0.078125( disk.nvme0c0n1.out‚31371.59423828125*Lwo“ß$‚(:ˆ ®¾æÔˆ¼£ý gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.834' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.5668 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0õm¾àà$‚(:ä ®¾æÔøê±ÿ disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚31862.13037109375# disk.dm-0.out‚31862.13037109375 disk.sda.in‚ 0.01171875 disk.sda3.out‚ 299.00390625 disk.drbd2266.out‚0 disk.dm-1.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 299.00390625 cpu‚25.36078781931175) disk./.usagePercent‚20.35198404200588 disk.rbd0.out‚0 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0# memory_percent‚91.2914514541626 network.recv‚ 2.810675e+06% disk./.usageGB‚303.25926971435547 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.drbd2265.out‚0 proc.cpu.threads‚17 disk.rbd1.out‚44.6875) proc.memory.availableMB‚ 1426.80859375! proc.memory.rssMB‚ 2645.234375 disk.sda.out‚ 299.00390625& disk.nvme0n1.out‚31862.13037109375 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125) prëûH9]oc.memory.percent‚16.14522933959961 network.sent‚ 2.4027832e+07 disk.drbd2266.in‚0‚k'Àžá$‚(:“ ½¾æÔ°·‚ý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚137.051' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.410199999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0BF(npâ$‚(:å ½¾æÔ§‹ƒÿ disk.dm-1.in‚ 0.01171875 cpu‚25.36022574524682 disk.rbd0.out‚0.0625 disk.rbd1.in‚0 disk.dm-1.out‚ 299.42578125& proc.memory.availableMB‚ 952.328125 disk.drbd2080.out‚0 disk.rbd0.in‚0# disk.dm-0.out‚31910.45849609375 disk.sda2.in‚0 network.recv‚ 2.814136e+06 disk.drbd2265.out‚0 disk.dm-0.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚31910.45849609375 disk.drbd2266.out‚0 disk.nvme0n1.in‚0.078125$ memory_percent‚94.18745040893555! proc.memory.rssMB‚ 2645.234375 network.sent‚ 2.4033082e+07 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 299.42578125 disk.sda.out‚ 299.42578125 disk.sda1.in‚0 disk.sda2.out‚0) proc.memory.percent‚16.14522933959961$ disk./.usageGB‚303.2643699645996 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0& disk.nvme0n1.out‚31910.45849609375 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.rbd1.out‚44.75* disk./.usagePercent‚20.352326324079158´µ÷52ã$- _wandbruntime‚7788  _runtime‚7788É~'³Éä$à ' lr-AdamW/pg1‚1.3926292255088996e-05 ' lr-AdamW/pg2‚1.3926292255088996e-05 trainer/global_step‚9749 % _timestamp‚1.7884526726481013e+09 _runtime‚7788.119691918 _step‚8003Õ¯Îå$È ' lr-AdamW/pg1‚1.3926292255088996e-05 ' lr-AdamW/pg2‚1.3926292255088996e-05 trainer/global_step‚9749 % _timestamp‚1.7884526726481013e+09 _runtime‚7788.119691918 _step‚800 (,\ê2æ$- _wandbruntime‚7788  _runtime‚7788Y,Ïç'ç$áO ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8176140980224 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚576197498746368 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 142109820 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 326400118 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8577568481792 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 % _timestamp‚1.7884526726488345e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 184270258 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 142109822 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 142109820 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8577398566400 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚553119011130368 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚46769448 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 184270258 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚576198173111808 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚568021357766144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚561265717701048 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚82921718 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚576197498746368 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 129690706 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 326380078 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚82921716 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚458 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚628782091847680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚561265046223932 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚628782285703168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 129691166 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚620204717221376 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8146035093564 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚7788.120666237 _step‚801 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚460 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚576198173111808 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚568021375769088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8176797342720 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 184290296 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚576197498746368 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 129690706 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 326380078 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚82921716 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚458 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚628782091847680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚561265046223932 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚628782285703168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 129691166 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚620204717221376 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8146035093564 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚7788.120666237 _step‚801 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚460 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚576198173111808 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚568021375769088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8176797342720 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 184290296 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 129703393 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8177616783872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚620266344018944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚46773969 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8578418752512 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚576254671213056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 184308776 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚568077728807936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚568077710804992 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 326432614 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚462 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚628844786711552 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚561320766789456 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 142123838 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚553173932300288 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 trainer/global_step‚9749 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚553173915260928 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚802 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8176960408064 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚576255345591808 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8147505966492 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8146851528528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 326432614 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 142123836 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 184308776 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚7788.817432853 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8176960408064 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 129703855 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚82929884 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚46773509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚628844588675072 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚460 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 $ _timestamp‚1.788452673345217e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚568077710804992 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 326412548 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 184288712 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚568077728807936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 326412548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 184308776 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚568077728807936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚568077710804992 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 326432614 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚462 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚628844786711552 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚561320766789456 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 142123838 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚553173932300288 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 trainer/global_step‚9749 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚553173915260928 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _step‚802 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8176960408064 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚576255345591808 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8147505966492 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8146851528528 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 326432614 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 142123836 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 184308776 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚7788.817432853 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8176960408064 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 129703855 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚82929884 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚46773509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 198036480 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚628844588675072 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚460 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 $ _timestamp‚1.788452673345217e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚568077710804992 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 326412548 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 184288712 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚568077728807936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 326412548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 129703393 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8177616783872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚620266344018944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚46773969 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8578418752512¢QzŽ2ì$- _wandbruntime‚7788  _runtime‚7788â(är\í$Ö 0loss/FRt4/assignment_loss‚2.6330815076828005 1loss/SRqqt2/detection_loss‚1.8905642168088392 1loss/SRqqt1/assignment_loss‚2.536754520734151 2loss/SRqqt2/assignment_loss‚1.4483868675094775 0loss/FBt1/assignment_loss‚0.4563296209976784 1loss/FBt4/assignment_loss‚0.15445515932515264 /loss/FRt4/detection_loss‚2.0678939227546964 /loss/FBt2/detection_loss‚1.5889427118441637 0loss/FRt1/assignment_loss‚4.2253112105222845 0loss/FRt2/assignment_loss‚3.1730040840382845 0loss/FBt2/assignment_loss‚0.4864909680669778 0loss/FBt3/assignment_loss‚0.1760984154782242 1loss/SRqqt4/detection_loss‚1.4264246212764524 &loss/total_loss‚45.391919125913084 epoch‚8 _runtime‚7788.819002796 /loss/FRt2/detection_loss‚2.1215448340155043 1loss/SRqqt3/detection_loss‚1.2107864702741304 _step‚803 trainer/global_step‚9749 1loss/SRqqt1/detection_loss‚2.5871497293313346 /loss/FBt1/detection_loss‚3.0493627429008483 0loss/FRt3/assignment_loss‚2.1790403856171503 2loss/SRqqt3/assignment_loss‚1.2706408239745846 .loss/FRt3/detection_loss‚2.476712135391103 /loss/FBt3/detection_loss‚0.8954962775591881 % _timestamp‚1.7884526733473904e+09 1loss/SRqqt4/assignment_loss‚1.437638576444186 .loss/FRt1/detection_loss‚3.128104959542935 .loss/FBt4/detection_loss‚2.771704363822937•â£Maî$Û 1loss/SRqqt1/detection_loss‚2.5871497293313346 /loss/FBt1/detection_loss‚3.0493627429008483 0loss/FRt3/assignment_loss‚2.1790403856171503 2loss/SRqqt3/assignment_loss‚1.2706408239745846 .loss/FRt3/detection_loss‚2.476712135391103 /loss/FBt3/detection_loss‚0.8954962775591881 % _timestamp‚1.7884526733473904e+09 1loss/SRqqt4/assignment_loss‚1.437638576444186 .loss/FRt1/detection_loss‚3.128104959542935 .loss/FBt4/detection_loss‚2.771704363822937 0loss/FRt4/assignment_loss‚2.6330815076828005 1loss/SRqqt2/detection_loss‚1.8905642168088392 1loss/SRqqt1/assignment_loss‚2.536754520734151 2loss/SRqqt2/assignment_loss‚1.4483868675094775 0loss/FBt1/assignment_loss‚0.4563296209976784 1loss/FBt4/assignment_loss‚0.15445515932515264 /loss/FRt4/detection_loss‚2.0678939227546964 /loss/FBt2/detection_loss‚1.5889427118441637 0loss/FRt1/assignment_loss‚4.2253112105222845 0loss/FRt2/assignment_loss‚3.1730040840382845 0loss/FBt2/assignment_loss‚0.4864909680669778 0loss/FBt3/assignment_loss‚0.1760984154782242 1loss/SRqqt4/detection_loss‚1.4264246212764524 &loss/total_loss‚45.391919125913084 epoch‚8 _runtime‚7788.819002796 /loss/FRt2/detection_loss‚2.1215448340155043 1loss/SRqqt3/detection_loss‚1.2107864702741304 _step‚803 trainer/global_step‚9749£ëíúÆ“ï$‚(:ˆ ̾æÔÍ̉ý gpu.0.gpu‚41.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.917' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.5834 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 7Zmð$‚(:â ̾æÔÍÂÿ$ disk./.usageGB‚303.2221870422363 disk.drbd2265.in‚0# disk.dm-0.out‚31965.56884765625& disk.nvme0n1.out‚31965.56884765625& proc.memory.availableMB‚ 952.296875) disk./.usagePercent‚20.34949539276553 disk.sda3.out‚ 300.1015625 disk.rbd0.in‚0 disk.rbd0.out‚0.0625 disk.sda2.in‚0 disk.rbd1.in‚0 disk.rbd1.out‚44.75$ memory_percent‚94.18764114379883 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0) proc.memory.percent‚16.14522933959961 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 300.1015625 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 proc.cpu.threads‚17 network.recv‚ 2.818787e+06 disk.sda1.in‚0 disk.sda1.out‚0 network.sent‚ 2.4039506e+07 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚31965.56884765625! proc.memory.rssMB‚ 2645.234375 cpu‚25.359629952060285 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 300.1015625 disk.drbd1716.in‚0Œ—Ä…‘ñ$‚(:† Û¾æÔòå»ý gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚181.91' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.382 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¼•¨|ò$‚(:ñ Û¾æÔŠúœÿ disk.drbd2266.out‚0 network.recv‚ 2.823928e+06 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚32022.14599609375 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.rbd1.out‚ 44.9619140625$ memory_percent‚94.18799877166748( proc.memory.availableMB‚ 952.23828125 disk.dm-1.in‚ 0.01171875 network.sent‚ 2.4147185e+07* disk./.usagePercent‚20.353143500128454 disk.sda3.out‚ 300.49609375 disk.dm-0.in‚0.078125 disk.sda.out‚ 300.49609375 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.nvme0n1.in‚0.078125) proc.memory.percent‚16.14522933959961 cpu‚25.35900606259321 proc.cpu.threads‚17 disk.drbd2080.in‚0& disk.nvme0n1.out‚32022.14599609375 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.dm-1.out‚ 300.49609375 disk.drbd2265.in‚0 disk.rbd0.out‚0.078125$ disk./.usageGB‚303.2765464782715# disk.dm-0.out‚32022.14599609375 disk.rbd1.in‚0! proc.memory.rssMB‚ 2645.234375U°¿2ó$- _wandbruntime‚7826  _runtime‚7826°ü$¶Çô$Á & lr-AdamW/pg1‚1.335939652889347e-05 & lr-AdamW/pg2‚1.335939652889347e-05 trainer/global_step‚9799 % _timestamp‚1.7884527105287035e+09 _runtime‚7826.000311754 _step‚804¯û¦MÌõ$Æ & lr-AdamW/pg2‚1.335939652889347e-05 trainer/global_step‚9799 % _timestamp‚1.7884527105287035e+09 _runtime‚7826.000311754 _step‚804 & lr-AdamW/pg1‚1.335939652889347e-05¤É)¼`2ö$- _wandbruntime‚7826  _runtime‚7826Ú ËGæ'÷$àO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8217126786048 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 185216348 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 328026970 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 130325938 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 130325501 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚437 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚83330116 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚623287230164480 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 142810622 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8620213865984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚579056136494592 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 185196310 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚570839027711488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚564051088091340 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 142810620 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 _step‚805 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 328006930 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚555864217658368 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚579056136494592 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8217783148544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚631907444030464 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚555864234697728 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 trainer/global_step‚9799 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚579056810860032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚46995385 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚435 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8620375392768 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 142810620 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8217783148544 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8186870432972 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 185196310 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚579056810860032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚570839009708544 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928XÄóì!ø$åO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚579056136494592 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 185196310 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚570839027711488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚564051088091340 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 142810620 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 _step‚805 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 328006930 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚555864217658368 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚579056136494592 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8217783148544 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚631907444030464 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚555864234697728 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 trainer/global_step‚9799 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚579056810860032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚46995385 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚435 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8620375392768 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 142810620 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8217783148544 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8186870432972 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 185196310 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚579056810860032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚570839009708544 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8217126786048 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 185216348 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 328026970 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 130325938 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 130325501 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚437 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚83330116 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚623287230164480 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 142810622 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8620213865984 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430¥,…1Û2ù$- _wandbruntime‚7826  _runtime‚7826”;-ñ&ú$ëM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8218602589696 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚555919121788928 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _step‚806 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚46999944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚579113308961280 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8218602589696 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚7826.719005817 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚570895362747392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚47000393 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 130338228 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8621233065984 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚555919138828288 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 328059466 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 208522240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 142824638 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚449 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 328059466 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 130338679 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚564107480134188 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚564106808656864 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚570895362747392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚451 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 185214764 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚631969929385984 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 184582144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚570895380750336 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚83338286 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚570895380750336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 328039400 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚579113983340032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 185234828 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 142824636 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 trainer/global_step‚9799 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 142824636 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8217946213888 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚631970137908224 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 142824638 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8621048483840 % _timestamp‚1.7884527112467926e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 185214764 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 185234828 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚579113983340032 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 185234828 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 142824636 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 trainer/global_step‚9799 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 142824636 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8217946213888 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚631970137908224 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 142824638 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8621048483840 % _timestamp‚1.7884527112467926e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 185214764 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 185234828 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚579113983340032 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚579113308961280 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8218602589696 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚7826.719005817 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚570895362747392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚47000393 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 130338228 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8621233065984 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚555919138828288 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 328059466 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 208522240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 142824638 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚449 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 328059466 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 130338679 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚564107480134188 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚564106808656864 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚570895362747392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚451 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 185214764 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚631969929385984¦°š2ü$- _wandbruntime‚7826  _runtime‚7826Kà·[ý$Õ _runtime‚7826.720316701 _step‚807 2loss/SRqqt1/assignment_loss‚2.8118542109013482 0loss/SRqqt1/detection_loss‚2.431287970613031 1loss/SRqqt2/detection_loss‚1.6121987051985882 /loss/FBt2/detection_loss‚2.7523386159113477 /loss/FBt3/detection_loss‚0.7593979412907517 /loss/FRt1/assignment_loss‚4.224508745329721 1loss/FBt2/assignment_loss‚0.35706275036228063 /loss/FRt3/detection_loss‚1.8333012943520495 1loss/SRqqt4/detection_loss‚1.7464297097527757 trainer/global_step‚9799 0loss/FBt4/assignment_loss‚0.7484579574292619 %loss/total_loss‚45.25395940560126 % _timestamp‚1.7884527112487907e+09 1loss/FBt3/assignment_loss‚0.14628506918747508 /loss/FRt3/assignment_loss‚1.770220801882122 /loss/FRt4/assignment_loss‚2.953809310650003 2loss/SRqqt3/assignment_loss‚1.3006339156283782 0loss/FBt1/assignment_loss‚0.7092845703182721 /loss/FRt1/detection_loss‚3.0345123388937543 /loss/FRt2/detection_loss‚2.1925045183071723 /loss/FRt2/assignment_loss‚2.811564321701343 /loss/FRt4/detection_loss‚2.1818107590079308 1loss/SRqqt3/detection_loss‚1.0145199118325343 epoch‚8 2loss/SRqqt2/assignment_loss‚1.0809797076508403 2loss/SRqqt4/assignment_loss‚1.3348894116451155 .loss/FBt1/detection_loss‚2.646539600456462 /loss/FBt4/detection_loss‚2.7995672672986984’z>`þ$Ú epoch‚8 2loss/SRqqt2/assignment_loss‚1.0809797076508403 2loss/SRqqt4/assignment_loss‚1.3348894116451155 .loss/FBt1/detection_loss‚2.646539600456462 /loss/FBt4/detection_loss‚2.7995672672986984 _runtime‚7826.720316701 _step‚807 2loss/SRqqt1/assignment_loss‚2.8118542109013482 0loss/SRqqt1/detection_loss‚2.431287970613031 1loss/SRqqt2/detection_loss‚1.6121987051985882 /loss/FBt2/detection_loss‚2.7523386159113477 /loss/FBt3/detection_loss‚0.7593979412907517 /loss/FRt1/assignment_loss‚4.224508745329721 1loss/FBt2/assignment_loss‚0.35706275036228063 /loss/FRt3/detection_loss‚1.8333012943520495 1loss/SRqqt4/detection_loss‚1.7464297097527757 trainer/global_step‚9799 0loss/FBt4/assignment_loss‚0.7484579574292619 %loss/total_loss‚45.25395940560126 % _timestamp‚1.7884527112487907e+09 1loss/FBt3/assignment_loss‚0.14628506918747508 /loss/FRt3/assignment_loss‚1.770220801882122 /loss/FRt4/assignment_loss‚2.953809310650003 2loss/SRqqt3/assignment_loss‚1.3006339156283782 0loss/FBt1/assignment_loss‚0.7092845703182721 /loss/FRt1/detection_loss‚3.0345123388937543 /loss/FRt2/detection_loss‚2.1925045183071723 /loss/FRt2/assignment_loss‚2.811564321701343 /loss/FRt4/detection_loss‚2.1818107590079308 1loss/SRqqt3/detection_loss‚1.0145199118325343§®î©’ÿ$‚(:‡ ê¾æÔî«—ý gpu.0.gpu‚1.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚123.941' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.7882 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0øŒ÷©y€%‚(:î ê¾æÔ³¦œ€ disk.rbd0.in‚0* disk./.usagePercent‚20.354232044164043$ disk./.usageGB‚303.2927665710449 disk.rbd1.in‚0' disk.nvme0c0n1.out‚32073.9423828125 disk.sda1.in‚0 network.recv‚ 2.829136e+06 disk.drbd1716.in‚0 disk.sda2.out‚0! proc.memory.rssMB‚ 2645.234375 disk.dm-1.out‚ 301.56640625 disk.sda.out‚ 301.56640625 disk.sda1.out‚0 disk.drbd2265.in‚0# memory_percent‚94.1910982131958( proc.memory.availableMB‚ 951.73046875 cpu‚25.358441478085723 network.sent‚ 2.4153878e+07 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 proc.cpu.threads‚17 disk.rbd1.out‚ 45.0556640625% disk.nvme0n1.out‚32073.9423828125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 301.56640625 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0" disk.dm-0.out‚32073.9423828125) proc.memory.percent‚16.14522933959961ÜL™“%‚(:ˆ ù¾æÔÚ€ý gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚180.222' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0444 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0%¦y‚%‚(:î ù¾æÔÂÛƒÿ disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.out‚0 disk.drbd1716.out‚0! proc.memory.rssMB‚ 2645.234375 disk.sda3.out‚ 302.10546875( proc.memory.availableMB‚ 952.17578125* disk./.usagePercent‚20.355837979426333 disk.rbd0.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125" disk.dm-0.out‚32119.4228515625 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 cpu‚25.35786183035632 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚32119.4228515625 network.sent‚ 2.4261748e+07 disk.sda.out‚ 302.10546875 disk.dm-1.in‚ 0.01171875$ memory_percent‚94.18838024139404) proc.memory.percent‚16.14522933959961 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.dm-1.out‚ 302.10546875 network.recv‚ 2.834372e+06$ disk./.usageGB‚303.3166961669922' disk.nvme0c0n1.out‚32119.4228515625 disk.rbd1.out‚ 45.2041015625 disk.drbd1716.in‚0Šü$“ƒ%‚(:ˆ ˆ¿æÔ»ˆîü gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚171.083' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.2166 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0É/o„%‚(:ä ˆ¿æÔ”þÿ network.sent‚ 2.4268281e+07 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 302.7265625 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.rbd0.in‚0 disk.dm-0.out‚32171.62109375 disk.nvme0n1.in‚0.078125$ memory_percent‚94.18835639953613 disk.sda.out‚ 302.7265625 network.recv‚ 2.839206e+06 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 302.7265625% disk.nvme0c0n1.out‚32171.62109375 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125) disk./.usagePercent‚20.35643370895851 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd1.in‚0) proc.memory.percent‚16.14522933959961 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.in‚0! proc.memory.rssMB‚ 2645.234375 disk.drbd1716.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 45.2041015625# disk.nvme0n1.out‚32171.62109375' proc.memory.availableMB‚ 952.1796875 proc.cpu.threads‚17$ disk./.usageGB‚303.3255729675293 disk.drbd1716.out‚0 disk.drbd2080.in‚0 cpu‚25.357270147191343¿NDeviceStatsMonitor.on_train_batch_ÒfiÙH%start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚566837129958748 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚83738516 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 186142400 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 186142400 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8258768954368 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 186122362 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚558609441225728 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 329633782 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 329653822 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚47223279 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚581914774242816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8663176848384 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚9849 _runtime‚7863.957159379 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚566837801435864 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚581914774242816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚444 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 130961353 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚47222837 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 143511422 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8228360210136 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8227705772380 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 186122362 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8663348860928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8258768954368 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚573656679653888 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚558609424186368 _step‚809 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚581915448608256 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚573656661650944 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚635033139848704 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8258112591872 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚635032943896064 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚626369790987776 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚573656661650944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 143511420 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚626369767047680 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8258112591872 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚558609424186368 _step‚809 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚581915448608256 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚573656661650944 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚635033139848704 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8258112591872 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚635032943896064 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚626369790987776 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚573656661650944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 143511420 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚626369767047680 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8258112591872 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚566837129958748 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚83738516 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 186142400 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 186142400 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8258768954368 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 186122362 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚442 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚558609441225728 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 329633782 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 329653822 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚47223279 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚581914774242816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8663176848384 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚9849 _runtime‚7863.957159379©yþÉ 2‹%- _wandbruntime‚7864  _runtime‚7864=)» ð&Œ%êM JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚635095461552640 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚573713014689792 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8664043767296 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8229176645308 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 186140816 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚626431417785344 $ _timestamp‚1.788452749189408e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 329686318 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8664203183616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚573713014689792 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚558664345356288 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚83746684 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚581972621088256 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚47227433 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8258932019712 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 186160880 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚581972621088256 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8259588395520 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚566893522001596 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚573713032692736 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 143525438 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 143525438 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 186140816 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 186160880 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 130974117 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚573713032692736 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚635095644909056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚626431441725440 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 329666252 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 143525436 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚558664328316928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8258932019712 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 329686318 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 329686318 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚558664345356288 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚83746684 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚581972621088256 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚47227433 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8258932019712 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 186160880 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚581972621088256 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8259588395520 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚566893522001596 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚573713032692736 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 143525438 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 143525438 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 186140816 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 186160880 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 130974117 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚573713032692736 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚635095644909056 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚626431441725440 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 329666252 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 143525436 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚558664328316928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8258932019712 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚584773411991040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚576474313593344 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 131598912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 144212220 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _step‚813 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 144212222 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 331280674 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 187048414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8299098397696 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8299754760192 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 187048414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8299098397696 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8299754760192 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚584773411991040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚576474313593344 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 131598912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 144212220 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 _step‚813 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 144212222 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 331280674 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8270011984716 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚638220987213824 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 187066868 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚576530684635136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚47456168 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 331293104 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 131611252 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚629513978608640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚584831258836480 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8269357546752 trainer/global_step‚9899 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚584830584457728 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚561409551884288 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 187086932 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚576530666632192 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 144226238 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 144226236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚576530684635136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8300574201344 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚569679563869004 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 331313170 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 331313170 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 144226236 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚561409534844928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚47456617 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚576530666632192 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8299917825536 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚584830584457728 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚561409551884288 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 187086932 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚576530666632192 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 144226238 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 144226236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚576530684635136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8300574201344 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚569679563869004 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 331313170 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 331313170 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 144226236 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚561409534844928 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚47456617 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚576530666632192 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8299917825536 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8270011984716 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚638220987213824 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 187066868 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚576530684635136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚47456168 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 331293104 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 131611252 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚629513978608640®Ï/2ž%- _wandbruntime‚7902  _runtime‚7902EäoXŸ%Ò 0loss/FRt4/assignment_loss‚2.5090457987278065 2loss/SRqqt4/assignment_loss‚1.0160192130896888 /loss/FBt1/detection_loss‚3.0250701268513995 .loss/FBt2/detection_loss‚2.375031287471453 0loss/FRt2/assignment_loss‚2.9463904327816435 0loss/FBt2/assignment_loss‚0.3401320346941172 /loss/FRt3/detection_loss‚1.8831833131438078 .loss/FRt4/detection_loss‚2.132736881362631 %loss/total_loss‚44.02058541837839 % _timestamp‚1.7884527868509579e+09 /loss/FRt1/assignment_loss‚3.694002410639887 0loss/FRt3/assignment_loss‚2.1620490370794783 /loss/FBt4/detection_loss‚3.0836197175085545 trainer/global_step‚9899 _runtime‚7902.322468415 2loss/SRqqt1/assignment_loss‚1.8669972799041055 2loss/SRqqt2/assignment_loss‚1.3327402702095557 /loss/FRt2/detection_loss‚2.2120043555895488 0loss/FBt3/assignment_loss‚0.1483289502876436 0loss/SRqqt1/detection_loss‚2.418239165436138 1loss/SRqqt2/detection_loss‚1.9169940839008408 0loss/SRqqt3/detection_loss‚1.459558415537079 /loss/FBt3/detection_loss‚0.7572220773177428 _step‚815 .loss/FRt1/detection_loss‚3.172543494597725 1loss/SRqqt3/assignment_loss‚1.474122809106484 1loss/FBt1/assignment_loss‚0.14713426087245504 0loss/FBt4/assignment_loss‚0.2809534941468428 0loss/SRqqt4/detection_loss‚1.666466508121764 epoch‚8B(;] %× /loss/FBt3/detection_loss‚0.7572220773177428 _step‚815 .loss/FRt1/detection_loss‚3.172543494597725 1loss/SRqqt3/assignment_loss‚1.474122809106484 1loss/FBt1/assignment_loss‚0.14713426087245504 0loss/FBt4/assignment_loss‚0.2809534941468428 0loss/SRqqt4/detection_loss‚1.666466508121764 epoch‚8 0loss/FRt4/assignment_loss‚2.5090457987278065 2loss/SRqqt4/assignment_loss‚1.0160192130896888 /loss/FBt1/detection_loss‚3.0250701268513995 .loss/FBt2/detection_loss‚2.375031287471453 0loss/FRt2/assignment_loss‚2.9463904327816435 0loss/FBt2/assignment_loss‚0.3401320346941172 /loss/FRt3/detection_loss‚1.8831833131438078 .loss/FRt4/detection_loss‚2.132736881362631 %loss/total_loss‚44.02058541837839 % _timestamp‚1.7884527868509579e+09 /loss/FRt1/assignment_loss‚3.694002410639887 0loss/FRt3/assignment_loss‚2.1620490370794783 /loss/FBt4/detection_loss‚3.0836197175085545 trainer/global_step‚9899 _runtime‚7902.322468415 2loss/SRqqt1/assignment_loss‚1.8669972799041055 2loss/SRqqt2/assignment_loss‚1.3327402702095557 /loss/FRt2/detection_loss‚2.2120043555895488 0loss/FBt3/assignment_loss‚0.1483289502876436 0loss/SRqqt1/detection_loss‚2.418239165436138 1loss/SRqqt2/detection_loss‚1.9169940839008408 0loss/SRqqt3/detection_loss‚1.459558415537079¯¼}’¡%‚(:‡ µ¿æÔ‰Äý gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚187.078' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.4156 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ér„g¢%‚(:Ü µ¿æÔëõÎÿ disk.sda.out‚304.5 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚32315.76708984375' proc.memory.availableMB‚ 952.0703125 network.sent‚ 2.4387886e+07) disk./.usagePercent‚20.35794671348587 disk.sda.in‚ 0.01171875 disk.drbd2265.in‚0% disk./.usageGB‚303.34811782836914 disk.rbd0.out‚0.078125 disk.rbd1.out‚ 45.6962890625 disk.drbd2080.in‚0) proc.memory.percent‚16.14522933959961 disk.drbd1716.out‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda3.in‚ 0.01171875$ memory_percent‚94.18902397155762 network.recv‚ 2.853242e+06 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.sda2.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125( disk.nvme0c0n1.out‚32315.76708984375 disk.sda3.out‚304.5! proc.memory.rssMB‚ 2645.234375 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚304.5 disk.drbd1716.in‚0 disk.drbd2265.out‚0 cpu‚25.355529808069843 disk.rbd0.in‚0# disk.dm-0.out‚32315.76708984375»“£%‚(:ˆ Ä¿æÔ”Ûíü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.232' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8464 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Tt|z¤%‚(:ï Ä¿æÔµýÿ proc.cpu.threads‚17 disk.drbd2265.out‚0* disk./.usagePercent‚20.358196832846822 disk.rbd0.out‚0.078125" disk.dm-0.out‚32373.1435546875% disk.nvme0n1.out‚32373.1435546875 network.sent‚ 2.4496777e+07 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 305.17578125 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda2.in‚0 disk.rbd1.in‚0' disk.nvme0c0n1.out‚32373.1435546875 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 cpu‚25.354956499439304% disk./.usageGB‚303.35184478759766 disk.sda1.in‚0 disk.rbd1.out‚ 45.6962890625 disk.drbd2265.in‚0! proc.memory.rssMB‚ 2645.234375 disk.drbd2266.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 305.17578125 disk.drbd2080.out‚0 disk.sda.out‚ 305.17578125 disk.sda2.out‚0 disk.drbd1716.in‚0( proc.memory.availableMB‚ 952.07421875) proc.memory.percent‚16.14522933959961 network.recv‚ 2.860398e+06 disk.sda3.in‚ 0.01171875# memory_percent‚94.1890001296997Ø3_“¥%‚(:ˆ Ó¿æÔÄýÌþ gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.013' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0026 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Âý+Êt¦%‚(:é Ó¿æÔè„Ø€% disk./.usageGB‚303.28775787353516 disk.rbd0.in‚0 disk.sda1.out‚0 disk.sda2.out‚0 disk.rbd1.out‚ 45.8681640625 disk.drbd1716.out‚0 network.recv‚ 2.863648e+06 disk.drbd2080.in‚0 disk.sda.in‚ 0.01171875) proc.memory.percent‚16.14522933959961' disk.nvme0c0n1.out‚32417.2177734375" disk.dm-0.out‚32417.2177734375 cpu‚25.35440325256695% disk.nvme0n1.out‚32417.2177734375 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.drbd2265.out‚0 network.sent‚ 2.4501911e+07 disk.sda.out‚ 305.640625 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.sda3.out‚ 305.640625 disk.drbd2266.out‚0 disk.drbd1716.in‚0( proc.memory.availableMB‚ 952.06640625! proc.memory.rssMB‚ 2645.234375 disk.drbd2080.out‚0 disk.rbd0.out‚0.078125 disk.nvme0n1.in‚0.078125$ memory_percent‚94.18904781341553 disk.dm-1.out‚ 305.640625* disk./.usagePercent‚20.353895906271585 disk.sda1.in‚0 disk.rbd1.in‚0Ôœ92§%- _wandbruntime‚7939  _runtime‚7939}5ýɨ%à _runtime‚7939.312077043 _step‚816 ' lr-AdamW/pg1‚1.1721669889316615e-05 ' lr-AdamW/pg2‚1.1721669889316615e-05 trainer/global_step‚9949 % _timestamp‚1.7884528238404162e+09Ù­wΩ%È ' lr-AdamW/pg1‚1.1721669889316615e-05 ' lr-AdamW/pg2‚1.1721669889316615e-05 trainer/global_step‚9949 % _timestamp‚1.7884528238404162e+09 _runtime‚7939.312077043 _step‚816°VhÁG2ª%- _wandbruntime‚7939  _runtime‚7939ÌÌ*ùÐ «%áO JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚434 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚84555318 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 144913022 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚579291983538688 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _runtime‚7939.313242142 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚47680940 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 187974466 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8749330195456 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚9949 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚47680508 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 144913020 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884528238412867e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 187974466 _step‚817 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 332907526 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 6544^Ï S37756 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8340084203520 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚579291965535744 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 144913022 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8340084203520 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚587632049739264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚632534840814080 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚641284171009536 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚572409885170680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚564099854281728 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 332887486 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128ÏbÐì'¬%æO F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 144913020 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8340084203520 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚579291965535744 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 144913022 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8340084203520 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚587632049739264 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚632534840814080 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚641284171009536 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚572409885170680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚564099854281728 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 332887486 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚434 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚84555318 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 144913022 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚579291983538688 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 _runtime‚7939.313242142 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚47680940 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 187974466 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8749330195456 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚9949 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚47680508 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 144913020 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884528238412867e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 187974466 _step‚817 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 332907526 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 132248550 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8750189262336 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 332940022 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚84563484 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8341560007168 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚587689896584704 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 332919956 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚84563486 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 187992920 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚572464934259088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚579348336577536 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 132248995 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚47685509 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚572465605736412 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚587689896584704 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚641346868364800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8310192886160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 332919956 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚564154741372928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 188012984 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 332940022 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7884528245532987e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8340903631360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _step‚818 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8341560007168 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚641346680814080 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8310847324124 wh·¯%ðM L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 132248550 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8750189262336 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 332940022 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚84563484 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚443 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8341560007168 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚587689896584704 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 332919956 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚445 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚84563486 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 187992920 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚572464934259088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚579348336577536 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 132248995 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚47685509 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚572465605736412 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚587689896584704 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚641346868364800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8310192886160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 332919956 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚564154741372928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 188012984 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 332940022 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7884528245532987e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8340903631360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _step‚818 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8341560007168 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚641346680814080 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8310847324124 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 144927036 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 trainer/global_step‚9949 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚564154758412288 _runtime‚7940.025346239 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚47685066²Äž2°%- _wandbruntime‚7940  _runtime‚7940¡qQÎY±%Ó 1loss/FBt3/assignment_loss‚0.20724324364806884 0loss/FBt4/assignment_loss‚0.5557835808237238 0loss/SRqqt2/detection_loss‚2.145104386103459 epoch‚8 /loss/FRt2/assignment_loss‚3.293761244416237 1loss/FBt2/assignment_loss‚0.45253647231893573 /loss/FRt1/detection_loss‚2.9324544031044533 /loss/FBt1/detection_loss‚2.8502605719999834 /loss/FBt2/detection_loss‚2.0442848865474974 1loss/SRqqt2/assignment_loss‚1.409683266129249 /loss/FBt1/assignment_loss‚0.251261107305254 1loss/SRqqt1/detection_loss‚2.3395965123176574 /loss/FBt3/detection_loss‚0.7964133529770953 /loss/FBt4/detection_loss‚3.2767300407091775 trainer/global_step‚9949 .loss/FRt2/detection_loss‚2.443796467036009 0loss/FRt3/assignment_loss‚1.5586463286165606 0loss/FRt4/assignment_loss‚3.2113069215646157 /loss/FRt4/detection_loss‚2.0724580832398853 0loss/SRqqt4/detection_loss‚1.497015917190799 _step‚819 1loss/SRqqt1/assignment_loss‚1.679824585467577 /loss/FRt3/detection_loss‚1.8741235118032546 _runtime‚7940.026704964 % _timestamp‚1.7884528245551867e+09 /loss/FRt1/assignment_loss‚4.072320933999686 2loss/SRqqt3/assignment_loss‚0.8501647625628667 0loss/SRqqt3/detection_loss‚1.182723118649686 &loss/total_loss‚44.501495480359054 2loss/SRqqt4/assignment_loss‚1.5040017818273217äÔ¯^²%Ø /loss/FRt2/assignment_loss‚3.293761244416237 1loss/FBt2/assignment_loss‚0.45253647231893573 /loss/FRt1/detection_loss‚2.9324544031044533 /loss/FBt1/detection_loss‚2.8502605719999834 /loss/FBt2/detection_loss‚2.0442848865474974 1loss/SRqqt2/assignment_loss‚1.409683266129249 /loss/FBt1/assignment_loss‚0.251261107305254 1loss/SRqqt1/detection_loss‚2.3395965123176574 /loss/FBt3/detection_loss‚0.7964133529770953 /loss/FBt4/detection_loss‚3.2767300407091775 trainer/global_step‚9949 .loss/FRt2/detection_loss‚2.443796467036009 0loss/FRt3/assignment_loss‚1.5586463286165606 0loss/FRt4/assignment_loss‚3.2113069215646157 /loss/FRt4/detection_loss‚2.0724580832398853 0loss/SRqqt4/detection_loss‚1.497015917190799 _step‚819 1loss/SRqqt1/assignment_loss‚1.679824585467577 /loss/FRt3/detection_loss‚1.8741235118032546 _runtime‚7940.026704964 % _timestamp‚1.7884528245551867e+09 /loss/FRt1/assignment_loss‚4.072320933999686 2loss/SRqqt3/assignment_loss‚0.8501647625628667 0loss/SRqqt3/detection_loss‚1.182723118649686 &loss/total_loss‚44.501495480359054 2loss/SRqqt4/assignment_loss‚1.5040017818273217 1loss/FBt3/assignment_loss‚0.20724324364806884 0loss/FBt4/assignment_loss‚0.5557835808237238 0loss/SRqqt2/detection_loss‚2.145104386103459 epoch‚8³»>Äž³%‚(:“ â¿æÔ¼ôºý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.737' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.547399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0äÏÌ*v´%‚(:ë â¿æÔ©‹¨€ disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0.078125) proc.memory.percent‚16.14522933959961 cpu‚25.353833442815002 proc.cpu.threads‚17* disk./.usagePercent‚20.355987743833865 disk.dm-1.out‚ 306.24609375 disk.drbd1716.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125' proc.memory.availableMB‚ 951.7578125 disk.rbd1.out‚ 45.8681640625 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda3.out‚ 306.24609375 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 network.sent‚ 2.4508658e+07 disk.sda2.out‚0$ disk./.usageGB‚303.3189277648926 disk.sda2.in‚0 disk.drbd2265.out‚0$ memory_percent‚94.19093132019043$ disk.nvme0n1.out‚32469.732421875 disk.drbd1716.in‚0& disk.nvme0c0n1.out‚32469.732421875! disk.dm-0.out‚32469.732421875 disk.sda.out‚ 306.24609375! proc.memory.rssMB‚ 2645.234375 network.recv‚ 2.868777e+06 disk.sda1.out‚0ëTn ‘µ%‚(:† ñ¿æÔܺý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚129.19' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.838 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0…½§7p¶%‚(:å ñ¿æÔ‡é›ÿ( disk.nvme0c0n1.out‚32519.54248046875$ memory_percent‚94.18940544128418' proc.memory.availableMB‚ 952.0078125 network.sent‚ 2.4616381e+07 disk.rbd0.out‚0.078125 disk.sda.out‚306.9375 disk.dm-1.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.drbd2266.out‚0 cpu‚25.353239307050103$ disk./.usageGB‚303.2825736999512 disk.rbd0.in‚0 disk.drbd2265.in‚0! proc.memory.rssMB‚ 2645.234375 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.sda3.in‚ 0.01171875) proc.memory.percent‚16.14522933959961 disk.rbd1.in‚0 disk.rbd1.out‚ 45.9306640625& disk.nvme0n1.out‚32519.54248046875 disk.sda3.out‚306.9375 network.recv‚ 2.874114e+06* disk./.usagePercent‚20.353547992032553 disk.dm-1.out‚306.9375 disk.drbd2080.in‚0 proc.cpu.threads‚17# disk.dm-0.out‚32519.54248046875 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125¦ì‡;2·%- _wandbruntime‚7976  _runtime‚7976ÂaFǸ%Á trainer/global_step‚9999 % _timestamp‚1.7884528614099529e+09 _runtime‚7976.881545089 _step‚820 & lr-AdamW/pg1‚1.119712708040318e-05 & lr-AdamW/pg2‚1.119712708040318e-05h—7̹%Æ & lr-AdamW/pg2‚1.119712708040318e-05 trainer/global_step‚9999 % _timestamp‚1.7884528614099529e+09 _runtime‚7976.881545089 _step‚820 & lr-AdamW/pg1‚1.119712708040318e-05´$ËE2º%- _wandbruntime‚7976  _runtime‚7976h&Tiæ'»%àO aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚635617377697280 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 334514338 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8381726371840 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 334534378 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚635617401637376 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚590490687487488 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚582109635481088 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚462 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚460 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚575195927038088 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 188900518 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚84963716 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚582109617478144 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 188920556 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚590491361852928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8381070009344 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚47909580 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚566845060809728 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚644409777812992µ…»Öë'¼%åO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 188920556 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚590491361852928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8381070009344 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚47909580 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚566845060809728 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚644409777812992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚635617377697280 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 334514338 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8381726371840 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 334534378 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚635617401637376 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚590490687487488 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚582109635481088 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚462 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚460 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚575195927038088 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 188900518 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚84963716 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚582109617478144 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872µÒÁM2½%- _wandbruntime‚7977  _runtime‚7977pÔ=Y¾%ëM _runtime‚7977.570847864 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 334546808 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 132886052 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8793433923072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 132885616 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 334566874 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 334566874 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 F@DeviceStatsMonitor.on_train_batch_end/oversize_alQ.¬˜ locations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚590548534332928 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚590547859954176 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 188939036 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8351682663532 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚582165970516992 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 188918972 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚566899964940288 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 145627838 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚84971884 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 145627836 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8381889437184 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚9999 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 145627836 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚575251647603820 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8793280798208 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚575250976126496 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚644472309233152 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚635679052375040 _step‚822 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8381889437184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8351028225568 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚582165970516992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚566899947900928 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 145627838 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414¨ eö&¿%ðM SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 145627838 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 _runtime‚7977.570847864 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 334546808 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 132886052 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8793433923072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 132885616 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 334566874 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 334566874 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚590548534332928 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚590547859954176 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 188939036 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8351682663532 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚582165970516992 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 188918972 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚566899964940288 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 145627838 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚84971884 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 145627836 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8381889437184 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚9999 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 145627836 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚575251647603820 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8793280798208 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚575250976126496 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚644472309233152 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚635679052375040 _step‚822 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8381889437184 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8351028225568 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚582165970516992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚566899947900928¶|?PN2À%- _wandbruntime‚7977  _runtime‚7977(Ð,…WÁ%Ñ /loss/FBt1/assignment_loss‚0.588290845042927 1loss/SRqqt3/detection_loss‚1.1233308981746843 0loss/SRqqt4/detection_loss‚1.750493000535404 .loss/FBt1/detection_loss‚2.166081450879574 /loss/FBt2/detection_loss‚1.8395456361273925 /loss/FRt3/detection_loss‚1.9263982844903418 1loss/SRqqt1/detection_loss‚2.5504873424768446 /loss/FBt4/detection_loss‚5.6476739048957825 % _timestamp‚1.7884528621005118e+09 /loss/FRt4/detection_loss‚2.1030310359597206 1loss/SRqqt2/detection_loss‚1.7104963448088049 0loss/FRt3/assignment_loss‚2.0593244977619336 .loss/FRt1/detection_loss‚3.387450767190833 _runtime‚7977.572309985 /loss/FRt1/assignment_loss‚3.449563057799088 0loss/FBt4/assignment_loss‚0.4605925679206848 epoch‚8 0loss/FRt4/assignment_loss‚2.9738236832618714 2loss/SRqqt3/assignment_loss‚0.9125869720536348 1loss/FBt3/assignment_loss‚0.23708650384476423 .loss/FRt2/detection_loss‚2.756044767358724 /loss/FBt3/detection_loss‚0.5407279711092512 $loss/total_loss‚46.0228475878438 /loss/FRt2/assignment_loss‚2.583949897219153 2loss/SRqqt1/assignment_loss‚1.8814098144570985 2loss/SRqqt4/assignment_loss‚1.1176967511099636 /loss/FBt2/assignment_loss‚0.638688774779439 trainer/global_step‚9999 2loss/SRqqt2/assignment_loss‚1.6180728185858766 _step‚823JÆM”\Â%Ö /loss/FRt2/assignment_loss‚2.583949897219153 2loss/SRqqt1/assignment_loss‚1.8814098144570985 2loss/SRqqt4/assignment_loss‚1.1176967511099636 /loss/FBt2/assignment_loss‚0.638688774779439 trainer/global_step‚9999 2loss/SRqqt2/assignment_loss‚1.6180728185858766 _step‚823 /loss/FBt1/assignment_loss‚0.588290845042927 1loss/SRqqt3/detection_loss‚1.1233308981746843 0loss/SRqqt4/detection_loss‚1.750493000535404 .loss/FBt1/detection_loss‚2.166081450879574 /loss/FBt2/detection_loss‚1.8395456361273925 /loss/FRt3/detection_loss‚1.9263982844903418 1loss/SRqqt1/detection_loss‚2.5504873424768446 /loss/FBt4/detection_loss‚5.6476739048957825 % _timestamp‚1.7884528621005118e+09 /loss/FRt4/detection_loss‚2.1030310359597206 1loss/SRqqt2/detection_loss‚1.7104963448088049 0loss/FRt3/assignment_loss‚2.0593244977619336 .loss/FRt1/detection_loss‚3.387450767190833 _runtime‚7977.572309985 /loss/FRt1/assignment_loss‚3.449563057799088 0loss/FBt4/assignment_loss‚0.4605925679206848 epoch‚8 0loss/FRt4/assignment_loss‚2.9738236832618714 2loss/SRqqt3/assignment_loss‚0.9125869720536348 1loss/FBt3/assignment_loss‚0.23708650384476423 .loss/FRt2/detection_loss‚2.756044767358724 /loss/FBt3/detection_loss‚0.5407279711092512 $loss/total_loss‚46.0228475878438·½Ô„W“Ã%‚(:ˆ €ÀæÔå‹ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚161.964' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚32.3928 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0|&¼­~Ä%‚(:ó €ÀæÔÛ©Éÿ disk.rbd0.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 45.9306640625 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 307.58984375 disk.drbd1716.out‚0 network.sent‚ 2.4622869e+07 network.recv‚ 2.878945e+06 disk.sda3.in‚ 0.01171875# disk.dm-0.out‚32583.37548828125 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125! proc.memory.rssMB‚ 2645.234375 proc.cpu.threads‚17% disk./.usageGB‚303.32011795043945 disk.sda.out‚ 307.58984375( disk.nvme0c0n1.out‚32583.37548828125$ memory_percent‚94.18962001800537* disk./.usagePercent‚20.356067618184547 disk.drbd1716.in‚0 cpu‚25.352669065042065 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0& disk.nvme0n1.out‚32583.37548828125 disk.drbd2080.out‚0( proc.memory.availableMB‚ 951.97265625) proc.memory.percent‚16.14522933959961 disk.sda3.out‚ 307.58984375•›‘Å%‚(:† ÀæÔƶ”ý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.26' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.252 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Tq–NrÆ%‚(:ç ÀæÔ disk.sda2.out‚0 disk.rbd1.out‚ 46.072265625 disk.dm-0.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2080.in‚0! proc.memory.rssMB‚ 2645.234375 network.recv‚ 2.884297e+06 disk.rbd0.out‚0.078125 disk.dm-1.out‚ 308.09375 proc.cpu.threads‚17& disk.nvme0n1.out‚32628.39501953125 disk.sda3.in‚ 0.01171875' proc.memory.availableMB‚ 951.7265625 disk.sda2.in‚0 network.sent‚ 2.4730521e+07 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0$ memory_percent‚94.19112205505371# disk.dm-0.out‚32628.39501953125 disk.dm-1.in‚ 0.01171875) proc.memory.percent‚16.14522933959961 cpu‚25.352129284901093 disk.rbd0.in‚0 disk.sda.out‚ 308.09375 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125( disk.nvme0c0n1.out‚32628.39501953125 disk.sda3.out‚ 308.09375) disk./.usagePercent‚20.35759521514136% disk./.usageGB‚303.34288024902344 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0ç€:ƒ“Ç%‚(:ˆ žÀæÔ¢£ý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.482' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8964 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÎкRyÈ%‚(:î žÀæÔ†Êòÿ proc.cpu.threads‚17 disk.drbd2265.in‚0( disk.nvme0c0n1.out‚32682.62939453125 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 308.71484375 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0* disk./.usagePercent‚20.358574955974895 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 308.71484375 network.sent‚ 2.4737235e+07 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚32682.62939453125 disk.sda2.in‚0 disk.rbd1.out‚ 46.158203125 cpu‚25.3515374404663 network.recv‚ 2.889198e+06 disk.drbd2265.out‚0 disk.rbd0.out‚0.078125! proc.memory.rssMB‚ 2645.234375# disk./.usageGB‚303.357479095459 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.drbd2080.out‚0# disk.dm-0.out‚32682.62939453125$ memory_percent‚94.18962001800537) proc.memory.percent‚16.14522933959961 disk.sda3.out‚ 308.71484375 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.sda2.out‚0 disk.rbd1.in‚0( proc.memory.availableMB‚ 951.97265625Ðqñy2É%- _wandbruntime‚8014  _runtime‚8014®ÎbuÊÊ%Ä ' lr-AdamW/pg1‚1.0683451361233388e-05 ' lr-AdamW/pg2‚1.0683451361233388e-05 trainer/global_step‚10049 % _timestamp‚1.7884528993197625e+09 _runtime‚8014.791346546 _step‚824À¬5ÏË%É _step‚824 ' lr-AdamW/pg1‚1.0683451361233388e-05 ' lr-AdamW/pg2‚1.0683451361233388e-05 trainer/global_step‚10049 % _timestamp‚1.7884528993197625e+09 _runtime‚8014.791346546¸LèZ¦2Ì%- _wandbruntime‚8014  _runtime‚8014Ó„ÐÍ%áO `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚584927287423488 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚577981297428380 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚638699914580480 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8422055815168 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8422055815168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 $ _timestamp‚1.788452899320421e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 189826570 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8835441305600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 189826570 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8422712177664 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 336161230 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 189846608 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚48139604 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8391047130012 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚593349999601152 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.aþEŒëçllocated‚85372118 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚48139141 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 336141190 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚593349999601152 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚85372116 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _step‚825 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 146314622 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 133511257 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8391701567768 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚593349325235712 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚584927269420544 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚577981968905496 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚593349325235712 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚647535355886080 trainer/global_step‚10049 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚569590250298368 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚584927287423488 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚465 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8422712177664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0ãjì'Î%æO @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _runtime‚8014.792263537 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 146314620 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 189826570 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8835441305600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 189826570 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8422712177664 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 336161230 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 189846608 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚48139604 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8391047130012 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚593349999601152 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚85372118 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚48139141 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 336141190 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚593349999601152 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚85372116 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _step‚825 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 146314622 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 133511257 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8391701567768 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚593349325235712 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚584927269420544 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚577981968905496 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚593349325235712 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚647535355886080 trainer/global_step‚10049 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚569590250298368 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚584927287423488 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚465 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8422712177664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚584927287423488 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚577981297428380 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚638699914580480 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8422055815168 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8422055815168 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 $ _timestamp‚1.788452899320421e+09 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 133523963 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚48144151 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚647597855472128 _step‚826 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚584983622459392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚48143679 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 146328638 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8422875243008 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 133524437 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 $ _timestamp‚1.788452900035808e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚584983640462336 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 189865088 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚593407172081152 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 146328636 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 336173660 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 336193726 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚578037689471228 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚474 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844uInç!Ñ%ðM H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 336193726 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 189845024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚584983640462336 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 189865088 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚593407172081152 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 146328636 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 336173660 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 336193726 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚578037689471228 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚474 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚593406497702400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 133523963 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚48144151 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚647597855472128 _step‚826 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚584983622459392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚48143679 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 146328638 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8422875243008 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 133524437 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/si{–egment.all.current‚844 $ _timestamp‚1.788452900035808e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 Bç'Ý%áO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚596207962983936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚587744921362944 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚580768010772904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚85780518 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8878676552704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8463041620992 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 147015422 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 147015422 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8463697983488 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 134150157 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚48370096 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚587744939365888 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚650661151956480 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 337768042 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 147015420 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 134150614 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8463697983488 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 147015420 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8431882469420 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 $ _timestamp‚1.788452937258598e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚48369641 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 190772660 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 190752622 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚587744939365888 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚641782475403776 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚596207962983936 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚85780516 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚641782451463680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8432536907176 _runtime‚8052.730458718 _step‚829 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚596208637349376 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8463041620992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚650660960198144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8878508734464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚10099 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚596208637349376 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 190772660 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 337788082 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 190752622 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928¥ùôuì'Þ%æO ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚650661151956480 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 337768042 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 147015420 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 134150614 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8463697983488 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 147015420 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8431882469420 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 $ _timestamp‚1.788452937258598e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚48369641 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 190772660 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 190752622 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚587744939365888 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚641782475403776 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚596207962983936 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚85780516 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚641782451463680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8432536907176 _runtime‚8052.730458718 _step‚829 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚596208637349376 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8463041620992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚650660960198144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 8878508734464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 trainer/global_step‚10099 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚596208637349376 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 190772660 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 337788082 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 190752622 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚596207962983936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 191758336 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚587744921362944 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚580768010772904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚85780518 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8878676552704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8463041620992 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 147015422 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 147015422 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8463697983488 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 134150157 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚48370096 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚587744939365888½ʱÉ}2ß%- _wandbruntime‚8053  _runtime‚8053Ε(Ç‹à%ìM K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 190771076 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚596265809829376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚587801274401792 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚48374146 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚48374589 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚587801292404736 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚587801292404736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 147029438 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 134162830 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 190791140 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8464517424640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 190791140 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚572390377996288 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8463861048832 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8879374118400 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 134163275 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 337800512 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚572390360956928 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚85788684 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚580823731338636 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 337820578 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚587801292404736 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚587801292404736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 147029438 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 134162830 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 190791140 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8464517424640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 190791140 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚572390377996288 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8463861048832 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8879374118400 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 134163275 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 337800512 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚572390360956928 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚85788684 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚580823731338636 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 337820578 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0¾Ž'4q2â%- _wandbruntime‚8053  _runtime‚8053$“x]ã%× /loss/FRt1/assignment_loss‚4.497101093331973 2loss/SRqqt1/assignment_loss‚1.8183772127543176 2loss/SRqqt3/assignment_loss‚0.9875599945442298 /loss/FRt3/detection_loss‚1.4583045453603924 0loss/FBt2/assignment_loss‚0.2845222871164779 1loss/SRqqt1/detection_loss‚2.4749190296445573 _step‚831 2loss/SRqqt2/assignment_loss‚1.3248726228167387 1loss/FBt1/assignment_loss‚0.49608709941547274 1loss/SRqqt3/detection_loss‚1.1140162070860733 1loss/SRqqt4/detection_loss‚1.3334787546685247 0loss/FBt4/assignment_loss‚0.6782096324423037 1loss/SRqqt2/detection_loss‚1.8679895116524263 epoch‚8 trainer/global_step‚10099 /loss/FRt1/detection_loss‚3.0061881666382155 0loss/FRt3/assignment_loss‚1.3260233155612289 1loss/FBt3/assignment_loss‚0.21565111380661878 /loss/FBt1/detection_loss‚2.8820406519449673 % _timestamp‚1.7884529379819698e+09 2loss/SRqqt4/assignment_loss‚1.1144045455889269 .loss/FBt4/detection_loss‚3.938029944896698 %loss/total_loss‚43.70606622261713 /loss/FRt4/assignment_loss‚2.909392476081848 /loss/FBt2/detection_loss‚2.0340215504169463 .loss/FBt3/detection_loss‚0.693716116737397 0loss/FRt2/assignment_loss‚2.5288442340162067 .loss/FRt4/detection_loss‚2.428450713555018 _runtime‚8053.453497526 /loss/FRt2/detection_loss‚2.2938654025395713꣙bä%Ü /loss/FRt4/assignment_loss‚2.909392476081848 /loss/FBt2/detection_loss‚2.0340215504169463 .loss/FBt3/detection_loss‚0.693716116737397 0loss/FRt2/assignment_loss‚2.5288442340162067 .loss/FRt4/detection_loss‚2.428450713555018 _runtime‚8053.453497526 /loss/FRt2/detection_loss‚2.2938654025395713 /loss/FRt1/assignment_loss‚4.497101093331973 2loss/SRqqt1/assignment_loss‚1.8183772127543176 2loss/SRqqt3/assignment_loss‚0.9875599945442298 /loss/FRt3/detection_loss‚1.4583045453603924 0loss/FBt2/assignment_loss‚0.2845222871164779 1loss/SRqqt1/detection_loss‚2.4749190296445573 _step‚831 2loss/SRqqt2/assignment_loss‚1.3248726228167387 1loss/FBt1/assignment_loss‚0.49608709941547274 1loss/SRqqt3/detection_loss‚1.1140162070860733 1loss/SRqqt4/detection_loss‚1.3334787546685247 0loss/FBt4/assignment_loss‚0.6782096324423037 1loss/SRqqt2/detection_loss‚1.8679895116524263 epoch‚8 trainer/global_step‚10099 /loss/FRt1/detection_loss‚3.0061881666382155 0loss/FRt3/assignment_loss‚1.3260233155612289 1loss/FBt3/assignment_loss‚0.21565111380661878 /loss/FBt1/detection_loss‚2.8820406519449673 % _timestamp‚1.7884529379819698e+09 2loss/SRqqt4/assignment_loss‚1.1144045455889269 .loss/FBt4/detection_loss‚3.938029944896698 %loss/total_loss‚43.70606622261713¿jk“å%‚(:ˆ ËÀæÔÚŒšý gpu.0.gpu‚37.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚178.951' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.7902 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0SKY—uæ%‚(:ê ËÀæÔŸæöÿ disk.dm-0.in‚0.078125 disk.dm-1.out‚ 310.43359375 disk.drbd1716.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 310.43359375 disk.rbd0.out‚0.078125 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0( disk.nvme0c0n1.out‚32806.57763671875 disk.drbd2266.in‚0 disk.rbd1.out‚ 46.3623046875& disk.nvme0n1.out‚32806.57763671875! proc.memory.rssMB‚ 2645.234375 disk.sda.out‚ 310.43359375 disk.sda1.in‚0 disk.rbd0.in‚0$ memory_percent‚94.19174194335938# proc.memory.availableMB‚951.625 proc.cpu.threads‚17 network.sent‚ 2.4856538e+07$ disk./.usageGB‚303.3867378234863 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0) proc.memory.percent‚16.14522933959961 disk.drbd2080.in‚0 network.recv‚ 2.902428e+06* disk./.usagePercent‚20.360538533762522 disk.sda2.in‚0 disk.drbd2266.out‚0 cpu‚25.349858809505# disk.dm-0.out‚32806.57763671875 disk.sda1.out‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125}ïØ“ç%‚(:ˆ ÚÀæÔÝ¿˜ý gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚125.839' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.1678 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0£bFÔyè%‚(:î ÚÀæÔ᪀! proc.memory.rssMB‚ 2645.234375 disk.drbd2266.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.out‚ 311.2421875 disk.rbd0.out‚0.078125 network.recv‚ 2.90864e+06 disk.sda.in‚ 0.01171875 disk.drbd2080.out‚0 disk.dm-0.in‚0.078125 disk.drbd1716.in‚0* disk./.usagePercent‚20.360969138435113 disk.drbd2266.out‚0# disk.dm-0.out‚32857.32861328125 disk.sda.out‚ 311.2421875 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚32857.32861328125) proc.memory.percent‚16.14522933959961 cpu‚25.349300825268735 disk.sda3.in‚ 0.01171875 proc.cpu.threads‚17 disk.rbd1.in‚0 network.sent‚ 2.4965501e+07( disk.nvme0c0n1.out‚32857.32861328125 disk.sda3.out‚ 311.2421875 disk.rbd0.in‚0 disk.sda1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0$ disk./.usageGB‚303.3931541442871 disk.drbd2080.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 46.3623046875$ memory_percent‚94.19028759002686( proc.memory.availableMB‚ 951.86328125/€·“é%‚(:ˆ éÀæÔ—íœý gpu.0.gpu‚27.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚155.939' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚31.1878 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0I£i}ê%‚(:ò éÀæÔúÊ«ÿ* disk./.usagePercent‚20.361561795876877 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.out‚ 311.96484375 proc.cpu.threads‚17% disk./.usageGB‚303.40198516845703 disk.sda.in‚ 0.01171875$ memory_percent‚94.19243335723877( proc.memory.availableMB‚ 951.51171875 disk.sda1.out‚0 disk.rbd0.out‚0.078125& disk.nvme0n1.out‚32898.10205078125 disk.drbd2265.in‚0 disk.drbd2265.out‚0! proc.memory.rssMB‚ 2645.234375 network.sent‚ 2.4970108e+07 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 311.96484375 disk.rbd0.in‚0 disk.dm-1.in‚ 0.01171875# disk.dm-0.out‚32898.10205078125 cpu‚25.34872313602017 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚32898.10205078125 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 network.recv‚ 2.912213e+06 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 311.96484375 disk.rbd1.out‚ 46.5185546875) proc.memory.percent‚16.145229339599612;•ò2ë%- _wandbruntime‚8090  _runtime‚8090ýE1Èì% trainer/global_step‚10149 % _timestamp‚1.7884529754727457e+09 _runtime‚8090.944348503 _step‚832 & lr-AdamW/pg1‚9.689127102130529e-06 & lr-AdamW/pg2‚9.689127102130529e-06'À-Íí%Ç & lr-AdamW/pg1‚9.689127102130529e-06 & lr-AdamW/pg2‚9.689127102130529e-06 trainer/global_step‚10149 % _timestamp‚1.7884529754727457e+09 _runtime‚8090.944348503 _step‚832À®¢>-2î%- _wandbruntime‚8090  _runtime‚8090úaª $$ï%áO UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚583553381163196 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚599066600732160 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚599067275097600 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚48599300 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 191698712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚590562573305344 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚590562591308288 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8473372246584 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚575080680393728 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8504683789312 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚644864988346880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 339394894 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 $ _timestamp‚1.788452975473476e+09 _runtime‚8090.945349172 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚599066600732160 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 191678674 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚590562573305344 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 134788675 ys?ZÃ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚653786783116288 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚10149 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8472717808828 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8921770829312 _step‚833 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872}ÌVì'ð%æO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚583553381163196 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚599066600732160 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚599067275097600 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚48599300 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 191698712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚590562573305344 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚590562591308288 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8473372246584 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚575080680393728 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8504683789312 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚644864988346880 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 339394894 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 $ _timestamp‚1.788452975473476e+09 _runtime‚8090.945349172 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚599066600732160 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 191678674 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚590562573305344 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 134788675 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚653786783116288 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 trainer/global_step‚10149 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚459 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8472717808828 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 8921770829312 _step‚833ÁæÙl2ñ%- _wandbruntime‚8091  _runtime‚8091¶ƒW5ò&ò%ìM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚599124447577600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚590618926344192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚86197086 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚653849125131264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 339447430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 147730236 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 191697128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8473534243792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8922486046720 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚575135584524288 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚590618944347136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚8091.656830067 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚86197084 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8504846854656 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚644926639084544 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚599123773198848 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚590618944347136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 147730238 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚10149 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8922643365888 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚468 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚575135567484928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 191697128 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8474188681756 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 339447430 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚644926663024640 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 147730236 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8504846854656 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 134801472 % _timestamp‚1.7884529761845937e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 147730238 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 191717192 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 134801004 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _step‚834 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 191717192 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8505503230464 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚48604386 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚583609773206044 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 339427364 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414•@ª÷&ó%ñM \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8474188681756 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 339447430 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚644926663024640 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 147730236 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8504846854656 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 134801472 % _timestamp‚1.7884529761845937e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 147730238 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 191717192 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 134801004 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 _step‚834 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 191717192 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8505503230464 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚48604386 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚583609773206044 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 339427364 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚599124447577600 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚590618926344192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚86197086 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚653849125131264 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 339447430 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 147730236 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 191697128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8473534243792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 8922486046720 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚575135584524288 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚590618944347136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _runtime‚8091.656830067 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚86197084 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8504846854656 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚644926639084544 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚599123773198848 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚590618944347136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 147730238 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚10149 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8922643365888 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚468 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚575135567484928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 191697128Âz@ÇÃ2ô%- _wandbruntime‚8091  _runtime‚8091Øe;ùWõ%Ñ 0loss/FBt1/assignment_loss‚0.6398996155088051 .loss/FRt2/detection_loss‚2.245961196043275 /loss/FRt4/detection_loss‚2.5175391828641294 1loss/SRqqt3/detection_loss‚1.2627515932697742 .loss/FBt3/detection_loss‚0.709395503748537 0loss/FRt2/assignment_loss‚3.4903410497036846 1loss/SRqqt1/assignment_loss‚1.918681811541319 1loss/FBt3/assignment_loss‚0.18959401488482386 /loss/FBt2/detection_loss‚2.7137520015239716 $loss/total_loss‚45.1367875438315 _step‚835 /loss/FRt3/assignment_loss‚1.351884264945984 1loss/SRqqt3/assignment_loss‚1.031478153480614 2loss/SRqqt4/assignment_loss‚1.3180992578066164 .loss/FRt3/detection_loss‚1.826435907781124 _runtime‚8091.658189914 0loss/FRt4/assignment_loss‚3.0308877408504484 2loss/SRqqt2/assignment_loss‚1.5395315180352904 1loss/FBt2/assignment_loss‚0.30278217047452854 .loss/FRt1/detection_loss‚2.800041319642748 1loss/SRqqt1/detection_loss‚2.6239525359123945 1loss/SRqqt2/detection_loss‚1.8738769836593092 /loss/FRt1/assignment_loss‚3.988681789806911 0loss/FBt4/assignment_loss‚0.1984918713569641 epoch‚8 .loss/FBt1/detection_loss‚2.400704232354959 .loss/FBt4/detection_loss‚3.664334312081337 % _timestamp‚1.7884529761863782e+09 trainer/global_step‚10149 1loss/SRqqt4/detection_loss‚1.4976895165539557<1«åuö%Ö _runtime‚8091.658189914 0loss/FRt4/assignment_loss‚3.0308877408504484 2loss/SRqqt2/assignment_loss‚b*ç1.5395315180352904 1loss/FBt2/assignment_loss‚0.30278217047452854 .loss/FRt1/detection_loss‚2.800041319642748 1loss/SRqqt1/detection_loss‚2.6239525359123945 1loss/SRqqt2/detection_loss‚1.8738769836593092 /loss/FRt1/assignment_loss‚3.988681789806911 0loss/FBt4/assignment_loss‚0.1984918713569641 epoch‚8 .loss/FBt1/detection_loss‚2.400704232354959 .loss/FBt4/detection_loss‚3.664334312081337 % _timestamp‚1.7884529761863782e+09 trainer/global_step‚10149 1loss/SRqqt4/detection_loss‚1.4976895165539557 0loss/FBt1/assignment_loss‚0.6398996155088051 .loss/FRt2/detection_loss‚2.245961196043275 /loss/FRt4/detection_loss‚2.5175391828641294 1loss/SRqqt3/detection_loss‚1.2627515932697742 .loss/FBt3/detection_loss‚0.709395503748537 0loss/FRt2/assignment_loss‚3.4903410497036846 1loss/SRqqt1/assignment_loss‚1.918681811541319 1loss/FBt3/assignment_loss‚0.18959401488482386 /loss/FBt2/detection_loss‚2.7137520015239716 $loss/total_loss‚45.1367875438315 _step‚835 /loss/FRt3/assignment_loss‚1.351884264945984 1loss/SRqqt3/assignment_loss‚1.031478153480614 2loss/SRqqt4/assignment_loss‚1.3180992578066164 .loss/FRt3/detection_loss‚1.826435907781124à oý"“÷%‚(:ˆ øÀæÔðåŽý gpu.0.gpu‚39.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚174.627' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.9254 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0þc¦úpø%‚(:å øÀæÔˆ–€ disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 312.640625 disk.drbd1716.out‚0 disk.sda3.out‚ 312.640625 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 proc.cpu.threads‚17' disk.nvme0c0n1.out‚32955.9013671875$ disk./.usageGB‚303.4228858947754 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125 cpu‚25.34819765069365 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.rbd0.out‚0.078125# memory_percent‚94.1927433013916' proc.memory.availableMB‚ 951.4609375) proc.memory.percent‚16.14522933959961 disk.nvme0c0n1.in‚0.078125 disk.rbd1.in‚0! proc.memory.rssMB‚ 2645.234375 disk.sda1.out‚0 network.recv‚ 2.917338e+06) disk./.usagePercent‚20.36296446115698 disk.drbd2266.in‚0 disk.sda.out‚ 312.640625 disk.nvme0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0" disk.dm-0.out‚32955.9013671875 disk.sda1.in‚0 disk.rbd1.out‚ 46.5185546875% disk.nvme0n1.out‚32955.9013671875 network.sent‚ 2.4976584e+07 disk.drbd1716.in‚0 disk.sda2.out‚0¸F¼“ù%‚(:ˆ ‡ÁæÔÅ‘ƒý gpu.0.gpu‚38.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.409' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Yd aú%‚(:Ö ‡ÁæÔ«Ì£€ disk.sda1.in‚0 disk.sda2.out‚0 proc.cpu.threads‚17 disk.dm-1.out‚313 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.rbd1.in‚0$ memory_percent‚94.19279098510742 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 network.sent‚ 2.5084449e+07* disk./.usagePercent‚20.363830790652848 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125# disk.dm-0.out‚33002.98388671875 disk.sda1.out‚0( disk.nvme0c0n1.out‚33002.98388671875% disk./.usageGB‚303.43579483032227 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.rbd1.out‚ 46.7763671875 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚33002.98388671875& proc.memory.availableMB‚ 951.453125! proc.memory.rssMB‚ 2645.234375 network.recv‚ 2.922466e+06 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda.out‚313) proc.memory.percent‚16.14522933959961 cpu‚25.347647459744937 disk.sda3.out‚313 disk.rbd0.out‚0.078125fåØR2û%- _wandbruntime‚8128  _runtime‚8128‘VóèÈü% & lr-AdamW/pg1‚9.208685777882703e-06 & lr-AdamW/pg2‚9.208685777882703e-06 trainer/global_step‚10199 % _timestamp‚1.7884530132523203e+09 _runtime‚8128.724006108 _step‚836Q7À›Íý%Ç & lr-AdamW/pg1‚9.208685777882703e-06 & lr-AdamW/pg2‚9.208685777882703e-06 trainer/global_step‚10199 % _timestamp‚1.7884530132523203e+09 _runtime‚8128.724006108 _step‚836Äú|s2þ%- _wandbruntime‚8128  _runtime‚8128¦òè'ÿ%âO ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚656912621250048 trainer/global_step‚10199 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚601925912845824 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 148417020 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8514207585992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚593380243250688 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 148417022 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚577825869882368 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 192624764 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 135427445 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚48830127 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8545013232640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚593380225247744 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 192604726 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚601925238480384 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8545669595136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚86597316 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚656912433686016 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 _runtime‚8128.724963996 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 192624764 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚647947549170176 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8513553148236 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚593380225247744 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 135427017 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚86597318d‡í'€&çO LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚647947549170176 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8513553148236 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚593380225247744 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 135427017 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚86597318 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚656912621250048 trainer/global_step‚10199 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚601925912845824 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 148417020 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8514207585992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚593380243250688 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 148417022 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚577825869882368 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 192624764 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 135427445 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚48830127 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8545013232640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚593380225247744 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 192604726 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚601925238480384 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8545669595136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚86597316 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚656912433686016 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 _runtime‚8128.724963996 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 192624764 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0ÅZ÷‘2&- _wandbruntime‚8129  _runtime‚8129}œ;EÇ‚&ìM OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 148431038 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 135440150 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 341054216 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚86605486 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 341074282 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚656975126927872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8545832660480 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 148431036 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 192623180 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8514369583200 % _timestamp‚1.7884530142280056e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 341074282 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8546489036288 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚440 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚593436578286592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 192643244 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚86605484 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 341054216 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚586395143596128 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚577880774012928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚577880791052288 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚48834226 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 192643244 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚648009175967744 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8515024021164 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚601983085325824 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚593436596289536 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 148431038 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 135439710 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚648009199907840 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMoniEkf5+ tor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8965927020032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚601982410947072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 trainer/global_step‚10199 _runtime‚8129.700367049 _step‚838 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚593436578286592 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚48834664 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚438 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚586395815073452 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8545832660480 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8546489036288 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚656974933085696 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928<Ûú¹÷&ƒ&ñM A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 trainer/global_step‚10199 _runtime‚8129.700367049 _step‚838 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚593436578286592 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚48834664 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚438 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚586395815073452 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8545832660480 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8546489036288 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚656974933085696 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 148431038 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 135440150 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 341054216 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚86605486 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 341074282 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚656975126927872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8545832660480 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 148431036 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 192623180 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8514369583200 % _timestamp‚1.7884530142280056e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 341074282 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8546489036288 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚440 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚593436578286592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 192643244 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚86605484 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 341054216 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚586395143596128 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚577880774012928 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚577880791052288 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚48834226 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 192643244 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚648009175967744 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8515024021164 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚601983085325824 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚593436596289536 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 148431038 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 135439710 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚648009199907840 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 8965927020032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚601982410947072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844ÆÆn:À2„&- _wandbruntime‚8129  _runtime‚8129r Æu\…&Ö 0loss/FRt4/assignment_loss‚3.7269500255584718 .loss/FRt4/detection_loss‚2.619868073426187 /loss/FBt3/detection_loss‚0.7247538052499294 1loss/FBt3/assignment_loss‚0.14392797726763543 epoch‚8 _step‚839 0loss/SRqqt3/assignment_loss‚1.24157334815356 /loss/FRt3/detection_loss‚1.5834032755415395 0loss/SRqqt1/detection_loss‚2.665534168481827 1loss/SRqqt4/detection_loss‚1.7089432483179527 /loss/FBt2/detection_loss‚2.5886312305927275 0loss/FBt2/assignment_loss‚0.3466166343539726 /loss/FBt1/detection_loss‚3.2013715744018554 %loss/total_loss‚46.25026583000169 /loss/FBt1/assignment_loss‚0.379286317527135 _runtime‚8129.701608522 2loss/SRqqt2/assignment_loss‚1.4414233864163575 2loss/SRqqt4/assignment_loss‚1.6467877823794097 /loss/FRt2/detection_loss‚2.1761689361598755 1loss/SRqqt3/detection_loss‚1.1843596156686544 trainer/global_step‚10199 % _timestamp‚1.7884530142300956e+09 1loss/FBt4/assignment_loss‚0.16852172023209278 /loss/FRt1/detection_loss‚3.0609579600136856 1loss/SRqqt2/detection_loss‚1.7834361257112545 /loss/FRt1/assignment_loss‚4.269919009044252 0loss/FRt3/assignment_loss‚2.1696216913889037 2loss/SRqqt1/assignment_loss‚1.9672214261154295 /loss/FBt4/detection_loss‚2.6418948570887246 0loss/FRt2/assignment_loss‚2.8090936409102545dÒMÕa†&Û /loss/FRt1/assignment_loss‚4.269919009044252 0loss/FRt3/assignment_loss‚2.1696216913889037 2loss/SRqqt1/assignment_loss‚1.9672214261154295 /loss/FBt4/detection_loss‚2.6418948570887246 0loss/FRt2/assignment_loss‚2.8090936409102545 0loss/FRt4/assignment_loss‚3.7269500255584718 .loss/FRt4/detection_loss‚2.619868073426187 /loss/FBt3/detection_loss‚0.7247538052499294 1loss/FBt3/assignment_loss‚0.14392797726763543 epoch‚8 _step‚839 0loss/SRqqt3/assignment_loss‚1.24157334815356 /loss/FRt3/detection_loss‚1.5834032755415395 0loss/SRqqt1/detection_loss‚2.665534168481827 1loss/SRqqt4/detection_loss‚1.7089432483179527 /loss/FBt2/detection_loss‚2.5886312305927275 0loss/FBt2/assignment_loss‚0.3466166343539726 /loss/FBt1/detection_loss‚3.2013715744018554 %loss/total_loss‚46.25026583000169 /loss/FBt1/assignment_loss‚0.379286317527135 _runtime‚8129.701608522 2loss/SRqqt2/assignment_loss‚1.4414233864163575 2loss/SRqqt4/assignment_loss‚1.6467877823794097 /loss/FRt2/detection_loss‚2.1761689361598755 1loss/SRqqt3/detection_loss‚1.1843596156686544 trainer/global_step‚10199 % _timestamp‚1.7884530142300956e+09 1loss/FBt4/assignment_loss‚0.16852172023209278 /loss/FRt1/detection_loss‚3.0609579600136856 1loss/SRqqt2/detection_loss‚1.7834361257112545ÇÀ’‡&‚(:‡ –ÁæÔý gpu.0.gpu‚5.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.917' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.5834 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ž±÷ºsˆ&‚(:è –ÁæÔ•Êïÿ network.recv‚ 2.927432e+06* disk./.usagePercent‚20.364368918489824 disk.sda.out‚ 313.703125 disk.drbd2265.out‚0 disk.rbd1.out‚ 46.7763671875 disk.dm-1.in‚ 0.01171875" disk.dm-0.out‚33064.5595703125( proc.memory.availableMB‚ 951.46484375 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125 disk.drbd2266.in‚0 network.sent‚ 2.509108e+07 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.out‚0% disk.nvme0n1.out‚33064.5595703125 cpu‚25.347102786117517 proc.cpu.threads‚17 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.dm-0.in‚0.078125) proc.memory.percent‚16.14522933959961 disk.sda.in‚ 0.01171875 disk.dm-1.out‚ 313.703125 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 313.703125$ memory_percent‚94.19271945953369! proc.memory.rssMB‚ 2645.234375$ disk./.usageGB‚303.4438133239746 disk.drbd1716.in‚0' disk.nvme0c0n1.out‚33064.5595703125ë–“‰&‚(:ˆ ¥ÁæÔ›ø«ý gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚125.789' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.1578 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0£ä ƒwŠ&‚(:ì ¥ÁæÔŸ…¥ÿ disk.drbd2265.in‚0 disk.sda2.in‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚33112.8134765625 cpu‚25.346561372791385* disk./.usagePercent‚20.365407029041165$ disk./.usageGB‚303.4592819213867 disk.dm-0.in‚0.078125 disk.sda.out‚ 314.1796875 disk.sda1.out‚0$ memory_percent‚94.19300556182861) proc.memory.percent‚16.14522933959961 disk.drbd2265.out‚0" disk.dm-0.out‚33112.8134765625 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 314.1796875 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚33112.8134765625 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 314.1796875 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875! proc.memory.rssMB‚ 2645.234375 network.sent‚ 2.5198691e+07 disk.rbd0.out‚0.078125 disk.rbd1.out‚ 46.8388671875( proc.memory.availableMB‚ 951.41796875 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 network.recv‚ 2.932767e+06 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0]ö÷ž‹&‚(:“ ´ÁæÔ¥‚…ý gpu.0.gpu‚37.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚183.834' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.766799999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0§ÙªItŒ&‚(:é ´ÁæÔÑñþþ disk.sda2.in‚0 disk.dm-1.out‚ 314.83203125 disk.nvme0c0n1.in‚0.078125! proc.memory.rssMB‚ 2645.234375) disk./.usagePercent‚20.36599635838499$ disk./.usageGB‚303.4680633544922 disk.dm-0.in‚0.078125 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0) proc.memory.percent‚16.14522933959961 cpu‚25.34602013594909 disk.sda2.out‚0 disk.rbd1.in‚0$ disk.nvme0n1.out‚33161.462890625 disk.drbd1716.in‚0# memory_percent‚94.1929578781128 network.sent‚ 2.5205163e+07! disk.dm-0.out‚33161.462890625 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875( proc.memory.availableMB‚ 951.42578125 network.recv‚ 2.937649e+06 disk.sda1.in‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 314.83203125 proc.cpu.threads‚17 disk.drbd2265.out‚0& disk.nvme0c0n1.out‚33161.462890625 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda.out‚ 314.83203125 disk.drbd1716.out‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125 disk.rbd1.out‚ 46.8388671875 disk.drbd2080.in‚0Ä€0t2&- _wandbruntime‚8166  _runtime‚8166{íÈŽ& _step‚840 & lr-AdamW/pg1‚8.739525931565641e-06 & lr-AdamW/pg2‚8.739525931565641e-06 trainer/global_step‚10249 % _timestamp‚1.7884530510583246e+09 _runtime‚8166.530005278^S4Í&Ç _step‚840 & lr-AdamW/pg1‚8.739525931565641e-06 & lr-AdamW/pg2‚8.739525931565641e-06 trainer/global_step‚10249 % _timestamp‚1.7884530510583246e+09 _runtime‚8166.530005278È;¨Ã’2&- _wandbruntime‚8166  _runtime‚81668püè'‘&âO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚651030062113280 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 342648598 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚596197877190144 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 193530778 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 136066104 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚589126136375128 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚49060386 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235328 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 149117822 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 193530778 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚651030086053376 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚580571076410368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8555042925400 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8554388487644 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚660038066375168 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚10249 _runtime‚8166.531108571 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 149117820 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚596197895193088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚596197877190144 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚596197895193088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8585999038464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 342668638 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 149117820 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚439 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9008159497216 % _timestamp‚1.7884530510592115e+09 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8585999038464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚87005718 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 193550816 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚604784550594048 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚87005716 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚660038245550592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚604783876228608 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9008004261888 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚589125464898012 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚604784550594048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8586655400960 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179175424 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚841 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚580571093449728 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 136065663 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8586655400960 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 149117822 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚604783876228608 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096ÁóË}v’&çO JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8585999038464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚87005718 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 193550816 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚604784550594048 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚87005716 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚660038245550592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/over9üšw size_segments.peak‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚604783876228608 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9008004261888 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚589125464898012 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚604784550594048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8586655400960 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 179175424 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚841 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚580571093449728 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 136065663 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8586655400960 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 149117822 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚604783876228608 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚651030062113280 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 342648598 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚596197877190144 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 193530778 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 136066104 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚589126136375128 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚49060386 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 155235328 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 149117822 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 193530778 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚651030086053376 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚580571076410368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8555042925400 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8554388487644 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚660038066375168 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚10249 _runtime‚8166.531108571 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 149117820 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚596197895193088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚596197877190144 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚596197895193088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8585999038464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 342668638 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 149117820 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚439 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9008159497216 % _timestamp‚1.7884530510592115e+09 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872ÉŸÔ:Ê2“&- _wandbruntime‚8167  _runtime‚8167dHÒ~ò&”&ìM C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 193549232 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚596254248231936 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚580625980540928 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 193549232 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 149131838 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚87013886 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚651091712850944 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8555859360572 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8586818466304 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚580625997580288 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 136078378 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 149131836 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚604841048695296 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚10249 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 136078811 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 342681068 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚49064494 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚660100766684672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49064925 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8555204922608 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 149131838 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚660100585425408 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 342701134 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 342701134 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚604841723074048 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 149131836 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚8167.228745319 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8587474842112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚87013884 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 193569296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8586818466304 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚433 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚589181856940860 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884530517566671e+09 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚596254230228992 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚651091736791040 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚842 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚596254248231936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9008872574464¦ò= ÷&•&ñM ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9008872574464 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 193549232 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚596254248231936 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚580625980540928 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 193549232 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 149131838 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚87013886 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚651091712850944 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8555859360572 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8586818466304 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚580625997580288 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 136078378 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 149131836 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚604841048695296 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚10249 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 136078811 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 342681068 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚49064494 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚660100766684672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49064925 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8555204922608 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 149131838 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚660100585425408 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 342701134 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 342701134 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚604841723074048 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 149131836 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚8167.228745319 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8587474842112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚87013884 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 193569296 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8586818466304 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚433 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚589181856940860 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884530517566671e+09 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚596254230228992 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚651091736791040 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚431 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚842 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚596254248231936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168ÊM‘2–&- _wandbruntime‚8167  _runtime‚8167ëE‚a—&Û /loss/FBt4/detection_loss‚3.3743369579315186 1loss/SRqqt4/detection_loss‚1.8639841795248806 /loss/FBt1/detection_loss‚3.2137248180806637 epoch‚8 2loss/SRqqt4/assignment_loss‚1.0523524286471448 1loss/FBt1/assignment_loss‚0.23900106223278428 1loss/FBt3/assignment_loss‚0.20402746863688867 /loss/FRt2/detection_loss‚2.5208852340124155 /loss/SRqqt2/detection_loss‚2.02543786675372 %loss/total_loss‚44.88100787768535 2loss/SRqqt1/assignment_loss‚2.2215320226308464 1loss/FBt4/assignment_loss‚0.25622164950306825 /loss/FBt2/detection_loss‚2.8297470957040787 % _timestamp‚1.7884530517582552e+09 0loss/FRt3/assignment_loss‚1.5314623511263303 0loss/FRt4/assignment_loss‚2.4971297244230906 2loss/SRqqt3/assignment_loss‚1.1345924407243728 .loss/FRt1/detection_loss‚2.982166720159126 1loss/SRqqt1/detection_loss‚2.2865175765913888 _runtime‚8167.229738554 _step‚843 0loss/FRt1/assignment_loss‚4.2742135777617944 2loss/SRqqt2/assignment_loss‚1.2510318221484726 /loss/FRt4/detection_loss‚2.2113257382065057 0loss/FRt2/assignment_loss‚3.0777692402663983 1loss/FBt2/assignment_loss‚0.36397778986906104 1loss/SRqqt3/detection_loss‚1.1003268283147078 /loss/FBt3/detection_loss‚0.7049664686847892 trainer/global_step‚10249 /loss/FRt3/detection_loss‚1.6642768157513015h%f˜&à 2loss/SRqqt1/assignment_loss‚2.2215320226308464 1loss/FBt4/assignment_loss‚0.25622164950306825 /loss/FBt2/detection_loss‚2.8297470957040787 % _timestamp‚1.7884530517582552e+09 0loss/FRt3/assignment_loss‚1.5314623511263303 0loss/FRt4/assignment_loss‚2.4971297244230906 2loss/SRqqt3/assignment_loss‚1.1345924407243728 .loss/FRt1/detection_loss‚2.982166720159126 1loss/SRqqt1/detection_loss‚2.2865175765913888 _runtime‚8167.229738554 _step‚843 0loss/FRt1/assignment_loss‚4.2742135777617944 2loss/SRqqt2/assignment_loss‚1.2510318221484726 /loss/FRt4/detection_loss‚2.2113257382065057 0loss/FRt2/assignment_loss‚3.0777692402663983 1loss/FBt2/assignment_loss‚0.36397778986906104 1loss/SRqqt3/detection_loss‚1.1003268283147078 /loss/FBt3/detection_loss‚0.7049664686847892 trainer/global_step‚10249 /loss/FRt3/detection_loss‚1.6642768157513015 /loss/FBt4/detection_loss‚3.3743369579315186 1loss/SRqqt4/detection_loss‚1.8639841795248806 /loss/FBt1/detection_loss‚3.2137248180806637 epoch‚8 2loss/SRqqt4/assignment_loss‚1.0523524286471448 1loss/FBt1/assignment_loss‚0.23900106223278428 1loss/FBt3/assignment_loss‚0.20402746863688867 /loss/FRt2/detection_loss‚2.5208852340124155 /loss/SRqqt2/detection_loss‚2.02543786675372 %loss/total_loss‚44.88100787768535Ë»Td“™&‚(:ˆ ÃÁæÔÁ÷¢ý gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.868' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9736 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 é–ªsš&‚(:è ÃÁæÔèùµÿ disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.drbd1716.out‚0! proc.memory.rssMB‚ 2645.234375 disk.dm-0.in‚0.078125 disk.drbd2080.in‚0& disk.nvme0c0n1.out‚33207.470703125) proc.memory.percent‚16.14522933959961 disk.rbd0.out‚0.078125! disk.dm-0.out‚33207.470703125 disk.sda1.in‚0 cpu‚25.345476083645547 proc.cpu.threads‚17$ disk./.usageGB‚303.4789237976074$ disk.nvme0n1.out‚33207.470703125 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2265.out‚0# memory_percent‚94.1913366317749 network.sent‚ 2.5210512e+07 disk.rbd1.in‚0 network.recv‚ 2.941209e+06 disk.sda3.out‚ 315.43359375 disk.rbd0.in‚0( proc.memory.availableMB‚ 951.69140625) disk./.usagePercent‚20.36672521183497 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.sda.out‚ 315.43359375 disk.sda1.out‚0 disk.rbd1.out‚ 47.05078125 disk.dm-1.out‚ 315.43359375 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125Sˆˆ²)›&‚(:“ ÒÁæÔÌë‹ý gpu.0.gpu‚ìo¨9u29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.525' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.504999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Óƒ^nœ&‚(:ã ÒÁæÔÒµ°€ disk.nvme0c0n1.in‚0.078125 disk.dm-1.out‚ 316.203125 disk.drbd2265.out‚0 disk.drbd2080.in‚0" disk.dm-0.out‚33266.7294921875 disk.sda2.in‚0 network.sent‚ 2.5319695e+07 network.recv‚ 2.9484e+06* disk./.usagePercent‚20.367402351763037 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.rbd1.out‚ 47.13671875 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.sda.out‚ 316.203125 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.rbd0.out‚0.078125 disk.rbd1.in‚0( proc.memory.availableMB‚ 951.69140625! proc.memory.rssMB‚ 2645.234375) proc.memory.percent‚16.14522933959961 cpu‚25.344900282368197 disk.drbd2080.out‚0 disk.sda3.out‚ 316.203125 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚33266.7294921875 disk.drbd1716.out‚0# disk./.usageGB‚303.489013671875' disk.nvme0c0n1.out‚33266.7294921875 disk.dm-0.in‚0.078125# memory_percent‚94.1913366317749 proc.cpu.threads‚17@úS2&- _wandbruntime‚8204  _runtime‚8204%?F¤Åž&¿ # _timestamp‚1.78845308862558e+09 _runtime‚ 8204.09717865 _step‚844 & lr-AdamW/pg1‚8.281745335388119e-06 & lr-AdamW/pg2‚8.281745335388119e-06 trainer/global_step‚10299±>ðûÊŸ&Ä # _timestamp‚1.78845308862558e+09 _runtime‚ 8204.09717865 _step‚844 & lr-AdamW/pg1‚8.281745335388119e-06 & lr-AdamW/pg2‚8.281745335388119e-06 trainer/global_step‚10299Ìlë_2 &- _wandbruntime‚8204  _runtime‚8204ÿô½ç'¡&áO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚49291482 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚663163898621952 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 344275450 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8626984844288 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚583316299977728 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 194476868 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚10299 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8627641206784 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚607643188342272 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 136705600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 $ _timestamp‚1.788453088626306e+09 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚607642513976832 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚663164081991680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚654112598996480 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8595223827052 _runtime‚8204.098143491 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8627641206784 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8626984844288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚87414116 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 194456830 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚599015529132544 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 149818620 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚49291029 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚607642513976832 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 149818620 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚87414118 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚607642513976832 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚663164081991680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚654112598996480 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8595223827052 _runtime‚8204.098143491 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8627641206784 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8626984844288 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚87414116 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 194456830 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚599015529132544 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 149818620 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚49291029 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚607642513976832 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 149818620 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚87414118 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8626984844288 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚583316299977728 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 194476868 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚10299 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8627641206784 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚607643188342272 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 136705600 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 $ _timestamp‚1.788453088626306e+09 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚654174273674240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚607699686443520 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 149832638 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 344307920 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚444 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚591967898808268 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 344327986 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚663226411243520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚663226596697088 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9052161509376 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 136717907 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 344307920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8596694699980 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚87422286 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚446 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚8204.788310979 _step‚846 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚10299 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8596040262016 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 % _timestamp‚1.7884530893161976e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 149832638 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49296067 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8628460647936°Sæ¥&ñM 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.˜˜¼&on_train_batch_end/active_bytes.small_pool.current‚ 656375808 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚663226411243520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚663226596697088 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9052161509376 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 136717907 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 344307920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8596694699980 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚87422286 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚446 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 _runtime‚8204.788310979 _step‚846 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 trainer/global_step‚10299 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8596040262016 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 % _timestamp‚1.7884530893161976e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 149832638 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49296067 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8628460647936 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚654174273674240 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚607699686443520 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 149832638 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 344307920 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚444 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚591967898808268 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 344327986 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605Îì‚ÙÊ2¦&- _wandbruntime‚8204  _runtime‚8204k·WÞZ§&Ô 2loss/SRqqt2/assignment_loss‚1.2966226776863665 2loss/SRqqt4/assignment_loss‚1.4402561927663868 /loss/FBt3/detection_loss‚0.7032651725624289 _runtime‚8204.789391869 0loss/FRt2/assignment_loss‚3.5257768478504445 2loss/SRqqt3/assignment_loss‚1.1511468662431947 1loss/FBt1/assignment_loss‚0.38227169816859174 /loss/FRt2/detection_loss‚2.2690646232560625 /loss/FRt1/assignment_loss‚3.596252231686204 2loss/SRqqt1/assignment_loss‚1.8953026278365044 0loss/FBt2/assignment_loss‚0.6791570156812667 0loss/SRqqt2/detection_loss‚1.932273860783019 0loss/SRqqt3/detection_loss‚1.212159378281622 epoch‚8 % _timestamp‚1.7884530893178868e+09 0loss/FBt3/assignment_loss‚0.2528643142814892 _step‚847 /loss/FRt1/detection_loss‚2.7679688621450356 %loss/total_loss‚44.14507375298817 trainer/global_step‚10299 /loss/FRt4/assignment_loss‚2.812254508987802 1loss/FBt4/assignment_loss‚0.06454140646320165 /loss/FRt3/detection_loss‚1.7785211338972051 0loss/SRqqt4/detection_loss‚1.686729146231865 /loss/FBt4/detection_loss‚3.4921703537305198 /loss/FRt3/assignment_loss‚1.218703920642535 .loss/FRt4/detection_loss‚2.062622774331296 1loss/SRqqt1/detection_loss‚2.6286658067857065 /loss/FBt1/detection_loss‚2.8929078451224735 .loss/FBt2/detection_loss‚2.403574487566948ÜD@l_¨&Ù 2loss/SRqqt4/assignment_loss‚1.4402561927663868 /loss/FBt3/detection_loss‚0.7032651725624289 _runtime‚8204.789391869 0loss/FRt2/assignment_loss‚3.5257768478504445 2loss/SRqqt3/assignment_loss‚1.1511468662431947 1loss/FBt1/assignment_loss‚0.38227169816859174 /loss/FRt2/detection_loss‚2.2690646232560625 /loss/FRt1/assignment_loss‚3.596252231686204 2loss/SRqqt1/assignment_loss‚1.8953026278365044 0loss/FBt2/assignment_loss‚0.6791570156812667 0loss/SRqqt2/detection_loss‚1.932273860783019 0loss/SRqqt3/detection_loss‚1.212159378281622 epoch‚8 % _timestamp‚1.7884530893178868e+09 0loss/FBt3/assignment_loss‚0.2528643142814892 _step‚847 /loss/FRt1/detection_loss‚2.7679688621450356 %loss/total_loss‚44.14507375298817 trainer/global_step‚10299 /loss/FRt4/assignment_loss‚2.812254508987802 1loss/FBt4/assignment_loss‚0.06454140646320165 /loss/FRt3/detection_loss‚1.7785211338972051 0loss/SRqqt4/detection_loss‚1.686729146231865 /loss/FBt4/detection_loss‚3.4921703537305198 /loss/FRt3/assignment_loss‚1.218703920642535 .loss/FRt4/detection_loss‚2.062622774331296 1loss/SRqqt1/detection_loss‚2.6286658067857065 /loss/FBt1/detection_loss‚2.8929078451224735 .loss/FBt2/detection_loss‚2.403574487566948 2loss/SRqqt2/assignment_loss‚1.2966226776863665ϟʪ’©&‚(:‡ áÁæÔÉãü gpu.0.gpu‚31.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.515' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.903 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0©ìêíoª&‚(:ä áÁæÔ£ƒ÷þ' disk.nvme0c0n1.out‚33312.9248046875 disk.sda3.in‚ 0.01171875" memory_percent‚94.191575050354 disk.sda.in‚ 0.01171875 disk.drbd2265.in‚0% disk./.usageGB‚303.49504470825195 disk.drbd2265.out‚0 disk.drbd2080.in‚0( proc.memory.availableMB‚ 951.65234375 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0) disk./.usagePercent‚20.36780709967467" disk.dm-0.out‚33312.9248046875 disk.drbd1716.in‚0 disk.sda3.out‚ 316.859375 disk.drbd2266.out‚0) proc.memory.percent‚16.14522933959961 cpu‚25.34437238371895 disk.sda1.out‚0 disk.sda2.out‚0 network.recv‚ 2.951626e+06 disk.rbd0.out‚0.078125 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 316.859375 disk.drbd1716.out‚0! proc.memory.rssMB‚ 2645.234375 proc.cpu.threads‚17 disk.sda.out‚ 316.859375 disk.sda1.in‚0 disk.rbd1.out‚ 47.28515625% disk.nvme0n1.out‚33312.9248046875 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 network.sent‚ 2.5324904e+07 disk.dm-1.in‚ 0.01171875ȧ‘«&‚(:† ðÁæÔÚ¤ùý gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚173.51' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.702 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0q¾y¬&‚(:î ðÁæÔÃüœÿ network.recv‚ 2.958533e+06 disk.sda3.out‚ 317.5546875# disk.dm-0.out‚33380.42236328125 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 47.28515625 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.out‚0( proc.memory.availableMB‚ 951.62890625 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0 disk.rbd1.in‚0& disk.nvme0n1.out‚33380.42236328125 disk.dm-1.out‚ 317.5546875 proc.cpu.threads‚17* disk./.usagePercent‚20.368593554819856 disk.rbd0.out‚0.078125) proc.memory.percent‚16.14522933959961 cpu‚25.343835208803544% disk./.usageGB‚303.50676345825195 disk.dm-0.in‚0.078125 disk.drbd2265.in‚0( disk.nvme0c0n1.out‚33380.42236328125 network.sent‚ 2.5434151e+07 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda.out‚ 317.5546875 disk.sda1.in‚0 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.in‚0$ memory_percent‚94.19171810150146! proc.memory.rssMB‚ 2645.234375˜ç™ “­&‚(:ˆ ÿÁæÔ̬íü gpu.0.gpu‚38.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚126.224' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.2448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ˆH†x®&‚(:í ÿÁæÔÉõ‡ÿ& disk.nvme0n1.out‚33426.77783203125 disk.drbd2266.in‚0$ memory_percent‚94.19171810150146 network.recv‚ 2.961557e+06# disk.dm-0.out‚33426.77783203125 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125( proc.memory.availableMB‚ 951.62890625* disk./.usagePercent‚20.369233061640386 disk.dm-0.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2265.out‚0( disk.nvme0c0n1.out‚33426.77783203125 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 network.sent‚ 2.5439289e+07 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.rbd1.in‚0 disk.rbd0.out‚0.078125 cpu‚25.343272682832257 proc.cpu.threads‚17 disk.rbd1.out‚ 47.44140625 disk.nvme0n1.in‚0.078125 disk.drbd2266.out‚0) proc.memory.percent‚16.14522933959961 disk.sda1.in‚0! proc.memory.rssMB‚ 2645.234375 disk.dm-1.out‚ 318.0546875 disk.drbd1716.out‚0 disk.sda3.out‚ 318.0546875 disk.rbd0.in‚0$ disk./.usageGB‚303.5162925720215 disk.sda.out‚ 318.0546875 disk.sda1.out‚0Î%É2¯&- _wandbruntime‚8241  _runtime‚8241_)\È°& & lr-AdamW/pg1‚7.835439390140258e-06 & lr-AdamW/pg2‚7.835439390140258e-06 trainer/global_step‚10349 % _timestamp‚1.7884531258916972e+09 _runtime‚8241.363318923 _step‚848¶¸O³Í±&Ç _runtime‚8241.363318923 _step‚848 & lr-AdamW/pg1‚7.835439390140258e-06 & lr-AdamW/pg2‚7.835439390140258e-06 trainer/global_step‚10349 % _timestamp‚1.7884531258916972e+09Ð1 à/2²&- _wandbruntime‚8241  _runtime‚8241T÷s–è'³&âO SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚49521255 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚601833181074944 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚594698220109944 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8636713604216 % _timestamp‚1.7884531258924077e+09 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚438 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚610501826090496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚610501151725056 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9094520965120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 195402920 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 150519422 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚610501151725056 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8668627012608 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚49521693 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8668627012608 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8667970650112 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 150519422 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8636059166460 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 137344211 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9094355244032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 trainer/global_step‚10349 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 150519420 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 195382882 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚666289680784896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚586061489466368 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚586061506505728 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚601833199077888 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚601833199077888 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚849 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 195402920 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 150519422 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚610501151725056 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8668627012608 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚49521693 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8668627012608 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8667970650112 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 150519422 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8636059166460 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 137344211 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9094355244032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 trainer/global_step‚10349 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 150519420 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 195382882 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚666289680784896 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚586061489466368 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚586061506505728 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚601833199077888 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚601833199077888 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚440 VHDeviceStatsMonitorî´*ØP.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚849 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9094520965120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 % _timestamp‚1.7884531265923488e+09 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚666352188172800 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 150533438 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 150533436 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 345954838 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8669446453760 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚49525734 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9095211907584 _step‚850 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8636875601424 _runtime‚8242.064532026 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚601889534113792 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 345934772 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 137356418 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8668790077952 trainer/global_step‚10349 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚610558998570496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49526171 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚610558998570496 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚610558324191744 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚601889534113792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 345934772 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 195421400 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 150533438 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚586116393596928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚666351998524928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 195401336 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚586116410636288 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066æ3á†÷&·&ñM L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 137356418 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚437 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8668790077952 trainer/global_step‚10349 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚610558998570496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚439 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49526171 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚610558998570496 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚610558324191744 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚601889534113792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 345934772 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 195421400 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 150533438 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚586116393596928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚666351998524928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 195401336 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚586116410636288 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8637530039388 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 137356857 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8669446453760 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 195421400 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚594753269198352 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 150533436 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚87830684 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 % _timestamp‚1.7884531265923488e+09 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚666352188172800 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 150533438 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 150533436 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 345954838 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8669446453760 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚49525734 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9095211907584 _step‚850 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8636875601424 _runtime‚8242.064532026 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚601889534113792 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 345934772 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚604650851020288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 % _timestamp‚1.7884531630362785e+09 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8676894505868 _runtime‚8278.508155323 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 151220220 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚88230916 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 196308934 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚613359789473280 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9137664490496 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚588806695994368 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 137983223 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 196328972 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 196308934 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚669415361193472 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚597484261977352 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚613359789473280 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8708956455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚480 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚660277696702976 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9137496672256 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚604650851020288 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚604650833017344 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚49752785 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8709612818432 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 151220222 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 trainer/global_step‚10399 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚669415169435136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚660277672762880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 196328972 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844ÀUÐ}í'Ä&çO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚88230918 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 151220220 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚49752785 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8709612818432 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 151220222 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 167818240 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 trainer/global_step‚10399 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚669415169435136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚660277672762880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 196328972 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚613360463838720 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚604650851020288 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 % _timestamp‚1.7884531630362785e+09 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8676894505868 _runtime‚8278.508155323 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 151220220 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚88230916 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 196308934 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚613359789473280 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9137664490496 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚588806695994368 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 137983223 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 196328972 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 196308934 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚669415361193472 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚597484261977352 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚613359789473280 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8708956455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚480 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚660277696702976 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9137496672256 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚604650851020288 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚604650833017344 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚465 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚588861600124928 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 196327388 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚88239084 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 196347452 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 137996016 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 % _timestamp‚1.7884531637288744e+09 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚467 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚613417636318720 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 347581690 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49757397 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9138349014528 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚604707186056192 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚613417636318720 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 151234238 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 347581690 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8710432259584 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 151234238 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 151234236 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚604707204059136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 196347452 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚660339347440640 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚88239086 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚604707204059136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚604707186056192 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚669477864260096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 347561624 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 347561624 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 151234236 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8677710940832 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944@| Ç&ñM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚597539311065760 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8678365378796 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 196327388 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8709775883776 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 _step‚854 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚465 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚‡oó588861600124928 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 196327388 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚88239084 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 196347452 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 137996016 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 % _timestamp‚1.7884531637288744e+09 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚467 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚613417636318720 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 347581690 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49757397 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9138349014528 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚604707186056192 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚613417636318720 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 151234238 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 347581690 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8710432259584 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 151234238 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 151234236 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚604707204059136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 196347452 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚660339347440640 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚88239086 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚604707204059136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚604707186056192 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚669477864260096 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 347561624 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 347561624 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 151234236 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8677710940832 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025Ö®(õ2È&- _wandbruntime‚8279  _runtime‚8279á¡O[É&Õ 0loss/FBt2/assignment_loss‚0.3962741475552313 2loss/SRqqt1/assignment_loss‚2.6458756742067635 2loss/SRqqt2/assignment_loss‚1.4690217724336045 /loss/FRt4/detection_loss‚2.2498031038305033 1loss/SRqqt1/detection_loss‚2.3852261900901794 2loss/SRqqt3/assignment_loss‚1.1307023352207113 /loss/FBt4/detection_loss‚2.4054329842329025 epoch‚8 1loss/FBt1/assignment_loss‚0.21884893376947426 .loss/FRt1/detection_loss‚3.339371913955325 /loss/FRt2/detection_loss‚2.0853452111283937 /loss/FRt3/detection_loss‚1.8268233977106152 trainer/global_step‚10399 _step‚855 2loss/SRqqt4/assignment_loss‚1.3086902824026057 1loss/FBt3/assignment_loss‚0.21602681225078818 /loss/SRqqt2/detection_loss‚1.72355266181486 1loss/SRqqt3/detection_loss‚0.9110676134760315 .loss/FRt3/assignment_loss‚1.60819603593982 0loss/FBt4/assignment_loss‚0.5201220157007356 $ _timestamp‚1.788453163730668e+09 0loss/FRt1/assignment_loss‚3.6581275917234874 /loss/FRt2/assignment_loss‚3.436866874496142 0loss/FRt4/assignment_loss‚2.6755804121494293 0loss/SRqqt4/detection_loss‚1.972031683456607 /loss/FBt1/detection_loss‚3.4953329960505166 &loss/total_loss‚44.431519530157665 /loss/FBt2/detection_loss‚2.0957031935453414 /loss/FBt3/detection_loss‚0.6574956930175956 _runtime‚8279.202199124–ãu+`Ê&Ú 0loss/FRt1/assignment_loss‚3.6581275917234874 /loss/FRt2/assignment_loss‚3.436866874496142 0loss/FRt4/assignment_loss‚2.6755804121494293 0loss/SRqqt4/detection_loss‚1.972031683456607 /loss/FBt1/detection_loss‚3.4953329960505166 &loss/total_loss‚44.431519530157665 /loss/FBt2/detection_loss‚2.0957031935453414 /loss/FBt3/detection_loss‚0.6574956930175956 _runtime‚8279.202199124 0loss/FBt2/assignment_loss‚0.3962741475552313 2loss/SRqqt1/assignment_loss‚2.6458756742067635 2loss/SRqqt2/assignment_loss‚1.4690217724336045 /loss/FRt4/detection_loss‚2.2498031038305033 1loss/SRqqt1/detection_loss‚2.3852261900901794 2loss/SRqqt3/assignment_loss‚1.1307023352207113 /loss/FBt4/detection_loss‚2.4054329842329025 epoch‚8 1loss/FBt1/assignment_loss‚0.21884893376947426 .loss/FRt1/detection_loss‚3.339371913955325 /loss/FRt2/detection_loss‚2.0853452111283937 /loss/FRt3/detection_loss‚1.8268233977106152 trainer/global_step‚10399 _step‚855 2loss/SRqqt4/assignment_loss‚1.3086902824026057 1loss/FBt3/assignment_loss‚0.21602681225078818 /loss/SRqqt2/detection_loss‚1.72355266181486 1loss/SRqqt3/detection_loss‚0.9110676134760315 .loss/FRt3/assignment_loss‚1.60819603593982 0loss/FBt4/assignment_loss‚0.5201220157007356 $ _timestamp‚1.788453163730668e+09×ü›ê‘Ë&‚(:† ¬ÂæÔ²ä©ý gpu.0.gpu‚38.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚138.12' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚27.624 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0öÃwÌ&‚(:ì ¬ÂæÔÙ¤Àÿ disk.drbd2265.out‚0 disk.sda.out‚ 319.74609375* disk./.usagePercent‚20.370346950421865$ disk./.usageGB‚303.5328903198242 disk.sda3.in‚ 0.01171875" disk.dm-0.out‚33580.7060546875 disk.rbd1.in‚0 disk.rbd1.out‚ 47.50390625' proc.memory.availableMB‚ 951.3515625 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875' disk.nvme0c0n1.out‚33580.7060546875 disk.drbd2266.in‚0 cpu‚25.341681607155696% disk.nvme0n1.out‚33580.7060546875 disk.dm-1.out‚ 319.74609375 disk.sda3.out‚ 319.74609375 disk.rbd0.in‚0 disk.drbd1716.in‚0 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875$ memory_percent‚94.19341087341309 disk.drbd2080.out‚0 disk.drbd2266.out‚0! proc.memory.rssMB‚ 2645.234375 network.recv‚ 2.977226e+06 disk.drbd1716.out‚0 disk.drbd2265.in‚0) proc.memory.percent‚16.14522933959961 proc.cpu.threads‚17 disk.sda1.in‚0 disk.sda1.out‚0 network.sent‚ 2.5560176e+07 disk.nvme0n1.in‚0.078125A\žÍ&‚(:“ »ÂæÔð±øü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚145.402' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.080399999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ä¨I½xÎ&‚(:í »ÂæÔã¹õÿ disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.sda2.out‚0 disk.dm-1.out‚ 320.3984375 disk.drbd2265.in‚0* disk./.usagePercent‚20.369013663183544 disk.sda1.out‚0 disk.rbd1.out‚ 47.6298828125 disk.rbd0.in‚0 disk.rbd1.in‚0% disk./.usageGB‚303.51302337646484 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0( proc.memory.availableMB‚ 951.53515625! proc.memory.rssMB‚ 2645.234375 network.sent‚ 2.5667749e+07" disk.dm-0.out‚33628.8583984375 disk.dm-1.in‚ 0.01171875$ memory_percent‚94.19229030609131 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 320.3984375 disk.dm-0.in‚0.078125 disk.drbd1716.in‚0 cpu‚25.341153468991596 proc.cpu.threads‚17 disk.drbd2266.out‚0 disk.sda.out‚ 320.3984375 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚33628.8583984375% disk.nvme0n1.out‚33628.8583984375) proc.memory.percent‚16.14522933959961 network.recv‚ 2.982019e+06!ÿ–%“Ï&‚(:ˆ ÊÂæÔ¨ƒâü gpu.0.gpu‚50.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚180.421' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0842 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÞF6 _Ð&‚(:Ô ÊÂæÔ©óüÿ) proc.memory.percent‚16.14522933959961 disk.rbd0.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 47.7158203125 disk.dm-1.in‚ 0.01171875* disk./.usagePercent‚20.370287556673926$ disk./.usageGB‚303.5320053100586 disk.sda3.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚321 network.sent‚ 2.5674533e+07 disk.drbd2266.in‚0# disk.dm-0.out‚33686.01318359375 cpu‚25.340599395920584 proc.cpu.threads‚17 disk.rbd0.out‚0.078125 disk.drbd2080.out‚0 disk.sda3.out‚321 disk.sda1.out‚0 disk.sda2.in‚0% proc.memory.availableMB‚ 951.28125! proc.memory.rssMB‚ 2645.234375 network.recv‚ 2.987398e+06 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125( disk.nvme0c0n1.out‚33686.01318359375 disk.sda.in‚ 0.01171875 disk.sda.out‚321& disk.nvme0n1.out‚33686.01318359375 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2080.in‚0$ memory_percent‚94.19384002685547í%/2Ñ&- _wandbruntime‚8315  _runtime‚8315ëÃìäÈÒ& _runtime‚8315.712500024 _step‚856 & lr-AdamW/pg1‚6.977621079710536e-06 & lr-AdamW/pg2‚6.977621079710536e-06 trainer/global_step‚10449 % _timestamp‚1.7884532002408848e+09µ(„ÿÍÓ&Ç _runtime‚8315.712500024 _step‚856 & lr-AdamW/pg1‚6.977621079710536e-06 & lr-AdamW/pg2‚6.977621079710536e-06 trainer/global_step‚10449 % _timestamp‚1.7884532002408848e+09Ø‚tŽð2Ô&- _wandbruntime‚8315  _runtime‚8315°š£ç'Õ&áO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚616219101586944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8749942261760 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚672541166692352 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8718384283032 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚616218427221504 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 197234986 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚607468502962688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚88639318 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚672540972836864 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 349156006 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚607468484959744 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 138622665 _runtime‚8315.713431372 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚591551919561728 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚88639316 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 197255024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8750598624256 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 197255024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚600270303844760 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚600269632367644 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚591551902522368 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚857 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 151921020 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 349176046 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8749942261760 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 197234986 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚672541166692352 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8718384283032 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚616218427221504 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 197234986 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚607468502962688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚88639318 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚672540972836864 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 349156006 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚607468484959744 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 138622665 _runtime‚8315.713431372 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚591551919561728 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚88639316 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 197255024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8750598624256 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 197255024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚600270303844760 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚600269632367644 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚591551902522368 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚857 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 151921020 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 349176046 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8749942261760 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 197234986 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚457 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 176193536 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 138635402 _step‚858 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚663421860383744 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9181602909184 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚591606806652928 _runtime‚8316.408255823 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚600325352933168 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 151935038 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚616276274066944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚672603663426560 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 349208542 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚672603463292928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 349208542 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚607524837998592 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚616275599688192 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚607524856001536 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚616276274066944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 % _timestamp‚1.7884532009361393e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 197273504 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8751418065408 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 138635873 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8719200718204 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8750761689600 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 151935036 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚469 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚663421884323840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚600326024410492 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8750761689600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 197253440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 151935038 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚88647484 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚471 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚88647486 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49988387 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200133632 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 349188476 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0N« ÷&Ù&ñM H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚471 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚88647486 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚49988387 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 200133632 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 349188476 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 176193536 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 138635402 _step‚858 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚663421860383744 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9181602909184 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚591606806652928 _runtime‚8316.408255823 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚600325352933168 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 151935038 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚616276274066944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚672603663426560 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 349208542 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚672603463292928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 349208542 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚607524837998592 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚616275599688192 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚607524856001536 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚616276274066944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 % _timestamp‚1.7884532009361393e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 197273504 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8751418065408 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 138635873 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8719200718204 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8750761689600 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 151935036 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚469 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚663421884323840 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚600326024410492 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8750761689600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 197253440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 151935038 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚88647484 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0Úëûò2Ú&- _wandbruntime‚8316  _runtime‚8316m¢£áXÛ&Ò 0loss/FRt3/assignment_loss‚1.1336825333890461 1loss/SRqqt1/detection_loss‚2.4346906685829164 /loss/FBt3/detection_loss‚0.6922850379274345 _runtime‚8316.409430661 1loss/FBt3/assignment_loss‚0.26394128555655005 /loss/FRt3/detection_loss‚1.9018138882127547 %loss/total_loss‚42.28427872683662 epoch‚8 trainer/global_step‚10449 1loss/SRqqt3/assignment_loss‚1.293089345915977 /loss/FRt1/assignment_loss‚4.032270457063403 0loss/FBt4/assignment_loss‚0.1739444945512029 1loss/SRqqt4/detection_loss‚1.3824537320813137 1loss/SRqqt1/assignment_loss‚1.545433224439621 2loss/SRqqt2/assignment_loss‚1.8179226669736883 /loss/FBt1/assignment_loss‚0.442139192663341 .loss/FRt1/detection_loss‚3.047453475849969 /loss/FBt2/detection_loss‚1.5930463802069426 _step‚859 0loss/FRt2/assignment_loss‚3.1532290837703605 0loss/FRt4/assignment_loss‚2.2023285381172015 /loss/FRt2/detection_loss‚2.3672718100058727 .loss/FBt4/detection_loss‚2.967678109804789 % _timestamp‚1.7884532009379466e+09 1loss/SRqqt4/assignment_loss‚1.296947511385626 1loss/SRqqt3/detection_loss‚1.2043448908423835 .loss/FBt1/detection_loss‚2.732427711670215 0loss/FBt2/assignment_loss‚0.6184485969715734 .loss/FRt4/detection_loss‚2.193956766925428 1loss/SRqqt2/detection_loss‚1.7934793239290063(Ÿ]Ü&× 0loss/FRt4/assignment_loss‚2.2023285381172015 /loss/FRt2/detection_loss‚2.3672718100058727 .loss/FBt4/detection_loss‚2.967678109804789 % _timestamp‚1.7884532009379466e+09 1loss/SRqqt4/assignment_loss‚1.296947511385626 1loss/SRqqt3/detection_loss‚1.2043448908423835 .loss/FBt1/detection_loss‚2.732427711670215 0loss/FBt2/assignment_loss‚0.6184485969715734 .loss/FRt4/detection_loss‚2.193956766925428 1loss/SRqqt2/detection_loss‚1.7934793239290063 0loss/FRt3/assignment_loss‚1.1336825333890461 1loss/SRqqt1/detection_loss‚2.4346906685829164 /loss/FBt3/detection_loss‚0.6922850379274345 _runtime‚8316.409430661 1loss/FBt3/assignment_loss‚0.26394128555655005 /loss/FRt3/detection_loss‚1.9018138882127547 %loss/total_loss‚42.28427872683662 epoch‚8 trainer/global_step‚10449 1loss/SRqqt3/assignment_loss‚1.293089345915977 /loss/FRt1/assignment_loss‚4.032270457063403 0loss/FBt4/assignment_loss‚0.1739444945512029 1loss/SRqqt4/detection_loss‚1.3824537320813137 1loss/SRqqt1/assignment_loss‚1.545433224439621 2loss/SRqqt2/assignment_loss‚1.8179226669736883 /loss/FBt1/assignment_loss‚0.442139192663341 .loss/FRt1/detection_loss‚3.047453475849969 /loss/FBt2/detection_loss‚1.5930463802069426 _step‚859 0loss/FRt2/assignment_loss‚3.1532290837703605Ûo!k“Ý&‚(:ˆ ÙÂæÔ‹·Šý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.362' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4724 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0]M¿wÞ&‚(:ì ÙÂæÔê¡ÿ disk.drbd2265.in‚0 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.dm-1.out‚ 321.390625 network.sent‚ 2.5679651e+07 disk.rbd0.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125( proc.memory.availableMB‚ 951.50390625 proc.cpu.threads‚17 disk.dm-0.in‚0.078125# disk.dm-0.out‚33733.38818359375 disk.drbd1716.in‚0! proc.memory.rssMB‚ 2645.234375 disk.sda.out‚ 321.390625 disk.drbd1716.out‚0) proc.memory.percent‚16.14522933959961 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 47.7783203125& disk.nvme0n1.out‚33733.38818359375 disk.rbd0.out‚0.078125 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 321.390625 cpu‚25.340083070577812) disk./.usagePercent‚20.37186968323553 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.drbd2080.in‚0 network.recv‚ 2.990633e+06% disk./.usageGB‚303.55558013916016 disk.dm-1.in‚ 0.01171875( disk.nvme0c0n1.out‚33733.38818359375$ memory_percent‚94.19248104095459`CEžß&‚(:“ èÂæÔ™”ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.017' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.203399999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡õߢgà&‚(:Ü èÂæÔ©Ùˆÿ disk.drbd2266.in‚0 disk.dm-0.out‚ 33790.078125 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.out‚ 322.26171875 disk.dm-0.in‚0.078125 disk.drbd2265.out‚0 network.sent‚ 2.5788512e+07 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda1.out‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17% disk./.usageGB‚303.57495498657227 disk.sda1.in‚0 disk.rbd1.out‚ 47.7783203125! disk.nvme0n1.out‚ 33790.078125 disk.dm-1.out‚ 322.26171875 disk.drbd2080.out‚0! proc.memory.availableMB‚951.5! proc.memory.rssMB‚ 2645.234375 disk.sda.in‚ 0.01171875 disk.sda.out‚ 322.26171875 disk.sda2.in‚0* disk./.usagePercent‚20.373169945501935 disk.rbd0.out‚0.078125 disk.dm-1.in‚ 0.01171875 network.recv‚ 2.997347e+06# memory_percent‚94.1925048828125) proc.memory.percent‚16.14522933959961# disk.nvme0c0n1.out‚ 33790.078125 disk.sda3.in‚ 0.01171875 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 cpu‚25.339563372005294W}!T2á&- _wandbruntime‚8352  _runtime‚8352JŸÁšÉâ&à ' lr-AdamW/pg1‚6.5662874825788626e-06 ' lr-AdamW/pg2‚6.5662874825788626e-06 trainer/global_step‚10499 % _timestamp‚1.7884532371586647e+09 _runtime‚ 8352.63028487 _step‚860–:w‚Îã&È ' lr-AdamW/pg2‚6.5662874825788626e-06 trainer/global_step‚10499 % _timestamp‚1.7884532371586647e+09 _runtime‚ 8352.63028487 _step‚860 ' lr-AdamW/pg1‚6.5662874825788626e-06ÜË䊋2ä&- _wandbruntime‚8352  _runtime‚8352S›®Xå&âO QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 152621822 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚619077064969728 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8790928067584 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚89047716 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9224047916544 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 198181076 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7884532371593878e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8790928067584 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 350802898 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚610286136902144 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚50213514 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 152621822 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8791584430080 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚610286154905088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚603055674235052 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚861 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚50213960 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 198181076 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 198161038 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 139261678 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚619077739335168 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚610286154905088 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 139261230 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚675666818385920 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚619077064969728 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 152621820 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚448 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚861 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚50213960 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 198181076 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 198161038 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 139261678 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚619077739335168 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚610286154905088 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 139261230 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚675666818385920 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚619077064969728 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 152621820 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚448 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚619077064969728 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8790928067584 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚89047716 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9224047916544 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 198181076 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7884532371593878e+09 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8790928067584 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 350802898 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚610286136902144 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚50213514 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 152621822 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8791584430080 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚610286154905088 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚603055674235052 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚862 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8791747495424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 350815328 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 198199556 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚610342507943936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚89055884 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚610342489940992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚89055886 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚675729341498368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 139274395 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 350835394 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9224920291328 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 152635836 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚603112066277900 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 152635836 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚666504397266944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚10499 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 198179492 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚50218509 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 139273948 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚594352013180928 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 350815328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9224762972160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8759381619648 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚8353.316539899 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 198199556 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚619134237436416 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚675729160239104 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 350835394 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚610342507943936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 % _timestamp‚1.7884532378444414e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚619134911815168 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 198179492 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚619134911815168 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 152635838=‰Áè<é&ñM SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 139273948 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚594352013180928 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 350815328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9224762972160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8759381619648 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚8353.316539899 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 198199556 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚619134237436416 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚675729160239104 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 350835394 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚610342507943936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 % _timestamp‚1.7884532378444414e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚619134911815168 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 198179492 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚619134911815168 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 152635838 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8792403871232 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚447 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚603111394800576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚862 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8791747495424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.curre‰z{[»nt‚ 671477324 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 350815328 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 198199556 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚610342507943936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚89055884 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚610342489940992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚89055886 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚675729341498368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 139274395 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 350835394 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚445 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9224920291328 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 152635836 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚603112066277900 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 152635836 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚666504397266944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚10499 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 198179492 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚50218509 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 trainer/global_step‚10549 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚621935702717952 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8832570235904 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8800054961848 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 153322622 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚621936377083392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚621935702717952 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚605841716102460 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9267248967680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚669525283412480 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 352429750 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8831913873408 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚605842387579576 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚621936377083392 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 139900732 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 139900285 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9267083246592 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚613103806847488 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚613103788844544 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 199087090 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚597042315578368 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚597042332617728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844£'¿í'ø&çO _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9267083246592 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚613103806847488 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚613103788844544 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 199087090 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚597042315578368 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚597042332617728 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 trainer/global_step‚10549 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚621935702717952 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8832570235904 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8800054961848 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 153322622 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚621936377083392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚621935702717952 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚605841716102460 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9267248967680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚669525283412480 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 352429750 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8831913873408 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚605842387579576 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚621936377083392 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 139900732 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 139900285áÚ~¦Õ2ù&- _wandbruntime‚8389  _runtime‚838932çÈú&ìM XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8832733301248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M>DeviceStatsMonitor.on_train_¾,±Å*&batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚866 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 139913061 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 _runtime‚8389.808033533 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚613160159886336 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 153336638 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚613160141883392 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚89464284 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 352462246 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 153336636 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 153336636 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚669586958090240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8833389677056 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 199125608 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 352462246 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8832733301248 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚621993549563392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚621992875184640 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 199125608 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚605898108145308 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884532743359795e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚678855061844480 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚678854868002304 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8800216959056 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚605897436667984 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 199105544 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8800871397020 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 352442180 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 199105544 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9267933852160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚597097236748288 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 352442180 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚89464284 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 352462246 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 153336636 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 153336636 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚669586958090240 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8833389677056 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 199125608 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 352462246 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8832733301248 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚621993549563392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 193842176 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚621992875184640 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 199125608 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚605898108145308 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 % _timestamp‚1.7884532743359795e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚456 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚678855061844480 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚678854868002304 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8800216959056 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚605897436667984 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 199105544 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8800871397020 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 352442180 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 199105544 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9267933852160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚597097236748288 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 352442180 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 169902080 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 _step‚866 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 139913061 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 _runtime‚8389.808033533 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚613160159886336 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944âFç 2ü&- _wandbruntime‚8389  _runtime‚8389®Õ3[ý&Õ 2loss/SRqqt3/assignment_loss‚1.0993497697015604 trainer/global_step‚10549 0loss/FRt4/assignment_loss‚2.6376692100269037 2loss/SRqqt4/assignment_loss‚1.4669301936023489 1loss/FBt4/assignment_loss‚0.03930129855750408 /loss/FRt2/detection_loss‚2.2925928313380632 1loss/SRqqt2/detection_loss‚1.8910485223239781 _runtime‚8389.808985149 /loss/FRt1/assignment_loss‚4.308430895209312 2loss/SRqqt1/assignment_loss‚1.9495577258938872 1loss/FBt3/assignment_loss‚0.16833285634530912 1loss/SRqqt1/detection_loss‚2.4094306085560775 .loss/FBt1/detection_loss‚3.060717113316059 /loss/FBt3/detection_loss‚0.6721077941440265 _step‚867 2loss/SRqqt2/assignment_loss‚1.7972895641412054 0loss/SRqqt3/detection_loss‚1.085398722688357 %loss/total_loss‚45.60403903695661 /loss/FRt2/assignment_loss‚3.042041832055801 /loss/FRt3/detection_loss‚1.7920618305603664 /loss/FBt2/detection_loss‚2.4331045936454427 epoch‚8 $ _timestamp‚1.788453274337484e+09 1loss/FBt1/assignment_loss‚0.26679845898132726 1loss/FBt2/assignment_loss‚0.23216049677726727 .loss/FRt1/detection_loss‚3.437046709656715 /loss/FRt4/detection_loss‚2.1903156258347556 /loss/FRt3/assignment_loss‚1.738185867998335 0loss/SRqqt4/detection_loss‚1.587404532565011 .loss/FBt4/detection_loss‚4.006761983036995êp9s`þ&Ú 1loss/FBt1/assignment_loss‚0.26679845898132726 1loss/FBt2/assignment_loss‚0.23216049677726727 .loss/FRt1/detection_loss‚3.437046709656715 /loss/FRt4/detection_loss‚2.1903156258347556 /loss/FRt3/assignment_loss‚1.738185867998335 0loss/SRqqt4/detection_loss‚1.587404532565011 .loss/FBt4/detection_loss‚4.006761983036995 2loss/SRqqt3/assignment_loss‚1.0993497697015604 trainer/global_step‚10549 0loss/FRt4/assignment_loss‚2.6376692100269037 2loss/SRqqt4/assignment_loss‚1.4669301936023489 1loss/FBt4/assignment_loss‚0.03930129855750408 /loss/FRt2/detection_loss‚2.2925928313380632 1loss/SRqqt2/detection_loss‚1.8910485223239781 _runtime‚8389.808985149 /loss/FRt1/assignment_loss‚4.308430895209312 2loss/SRqqt1/assignment_loss‚1.9495577258938872 1loss/FBt3/assignment_loss‚0.16833285634530912 1loss/SRqqt1/detection_loss‚2.4094306085560775 .loss/FBt1/detection_loss‚3.060717113316059 /loss/FBt3/detection_loss‚0.6721077941440265 _step‚867 2loss/SRqqt2/assignment_loss‚1.7972895641412054 0loss/SRqqt3/detection_loss‚1.085398722688357 %loss/total_loss‚45.60403903695661 /loss/FRt2/assignment_loss‚3.042041832055801 /loss/FRt3/detection_loss‚1.7920618305603664 /loss/FBt2/detection_loss‚2.4331045936454427 epoch‚8 $ _timestamp‚1.788453274337484e+09ã1Sªžÿ&‚(:“ ¤ÃæÔö¥êý gpu.0.gpu‚41.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚127.293' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.458600000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0F4Êno€'‚(:ä ¤ÃæԢܔ€ cpu‚25.337436024979826 disk.drbd2080.out‚0& disk.nvme0c0n1.out‚33988.994140625 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 network.sent‚ 2.5913801e+07 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda.out‚ 325.0859375$ disk.nvme0n1.out‚33988.994140625 disk.dm-1.in‚ 0.01171875! proc.memory.rssMB‚ 2645.234375) proc.memory.percent‚16.14522933959961) disk./.usagePercent‚20.37644146578199$ disk./.usageGB‚303.6237030029297 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 325.0859375 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0.078125! disk.dm-0.out‚33988.994140625% proc.memory.availableMB‚ 951.03125 network.recv‚ 3.015522e+06 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.sda1.out‚0 disk.sda2.out‚0$ memory_percent‚94.19536590576172 disk.drbd1716.in‚0 disk.sda3.out‚ 325.0859375 disk.drbd2266.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 48.060546875 proc.cpu.threads‚17 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0;Ÿp‘'‚(:† ³ÃæÔùøŸý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚177.38' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.476 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0òt;dr‚'‚(:ç ³ÃæÔÊ»ûÿ* disk./.usagePercent‚20.376791428081297$ disk./.usageGB‚303.6289176940918 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.sda1.out‚0 proc.cpu.threads‚17 disk.sda3.out‚ 325.6328125 disk.sda1.in‚0$ disk.nvme0n1.out‚34033.048828125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 cpu‚25.336926686350537 network.recv‚ 3.020654e+06& disk.nvme0c0n1.out‚34033.048828125 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0! disk.dm-0.out‚34033.048828125 disk.sda.out‚ 325.6328125 disk.drbd2265.in‚0 disk.rbd1.out‚ 48.318359375 disk.dm-1.in‚ 0.01171875' proc.memory.availableMB‚ 951.0234375 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 325.6328125! proc.memory.rssMB‚ 2645.234375 network.sent‚ 2.6021419e+07 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 disk.sda2.in‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0$ memory_percent‚94.19541358947754) proc.memory.percent‚16.14522933959961zƒF2ƒ'- _wandbruntime‚8426  _runtime‚8426චùÈ„' trainer/global_step‚10599 % _timestamp‚1.7884533112117212e+09 _runtime‚8426.683323049 _step‚868 & lr-AdamW/pg1‚5.779199993143309e-06 & lr-AdamW/pg2‚5.779199993143309e-06F {MÍ…'Ç & lr-AdamW/pg1‚5.779199993143309e-06 & lr-AdamW/pg2‚5.779199993143309e-06 trainer/global_step‚10599 % _timestamp‚1.7884533112117212e+09 _runtime‚8426.683323049 _step‚868äæ‚(™2†'- _wandbruntime‚8426  _runtime‚8426ßSJE‡‡'áO QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 200013142 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 154023422 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚50673784 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚89864518 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚10599 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚681918164752384 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 354036562 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 354056602 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8873556041728 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 140538752 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 154023420 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 200033180 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8872899679232 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 200033180 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚624795014831616 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚624794340466176 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚615921440786944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚624795014831616 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚624794340466176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚89864516 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚608627757969868 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8840235863500 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚8426.684284794 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 200013142 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚615921458789888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚672607820295680 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9310320516608 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 154023422 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚50673784 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚89864518 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 trainer/global_step‚10599 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚681918164752384 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 354036562 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 354056602 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚450 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8873556041728 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 140538752 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 154023420 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 200033180 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8872899679232 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 200033180 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚624795014831616 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚624794340466176 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚615921440786944 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚624795014831616 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚624794340466176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚89864516 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚608627757969868 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8840235863500 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚452 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚8426.684284794 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 200013142 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚615921458789888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚672607820295680 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9310320516608 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 354089098 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8841706736428 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚615977811828736 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8873719107072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 _runtime‚8427.405082973 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8873719107072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚608683478535392 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 200031596 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚624852187311616 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 % _timestamp‚1.7884533119328544e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 154037438 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 140551020 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚599842426236928 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 200031596 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 354089098 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚615977811828736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 154037436 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414F¢m"‹'ñM VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚681980676197376 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 354089098 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8841706736428 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚615977811828736 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8873719107072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 _runtime‚8427.405082973 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8873719107072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚608683478535392 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚453 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 200031596 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚624852187311616 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 % _timestamp‚1.7884533119328544e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 154037438 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 140551020 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚599842426236928 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 2¾Š00031596 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 354089098 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚615977811828736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 154037436 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚681980484452352 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚599842443276288 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 trainer/global_step‚10599æO¼nï2Œ'- _wandbruntime‚8427  _runtime‚8427udZ'Ô 1loss/SRqqt4/assignment_loss‚1.542879479787042 0loss/FBt4/assignment_loss‚0.3477600751866703 0loss/SRqqt3/detection_loss‚1.078723198389762 /loss/FBt2/detection_loss‚2.2703917175531387 /loss/FBt3/detection_loss‚0.9393990428852184 0loss/FRt3/assignment_loss‚1.8287640649538774 /loss/FRt3/detection_loss‚1.8046865372990186 1loss/SRqqt2/detection_loss‚1.9297436057608954 .loss/FBt4/detection_loss‚3.706367090344429 trainer/global_step‚10599 2loss/SRqqt3/assignment_loss‚1.1680677806677884 /loss/FBt1/detection_loss‚3.1960037549336753 /loss/FRt4/assignment_loss‚2.780461304462873 .loss/FRt2/detection_loss‚2.055902916295775 epoch‚8 0loss/FRt2/assignment_loss‚3.6177640916972327 2loss/SRqqt1/assignment_loss‚1.9058939119180043 2loss/SRqqt2/assignment_loss‚1.8698874762547868 0loss/FBt2/assignment_loss‚0.5648097153364992 .loss/FRt4/detection_loss‚2.115172511109939 1loss/SRqqt4/detection_loss‚1.4920798126728303 _runtime‚8427.406028868 0loss/FBt3/assignment_loss‚0.2436596647441604 0loss/SRqqt1/detection_loss‚2.645922021071116 1loss/FBt1/assignment_loss‚0.41904255911059507 %loss/total_loss‚45.87436467138957 % _timestamp‚1.7884533119345138e+09 0loss/FRt1/assignment_loss‚3.6012694316032605 /loss/FRt1/detection_loss‚2.7497129073509803 _step‚871ó+ñ_Ž'Ù 0loss/FBt2/assignment_loss‚0.5648097153364992 .loss/FRt4/detection_loss‚2.115172511109939 1loss/SRqqt4/detection_loss‚1.4920798126728303 _runtime‚8427.406028868 0loss/FBt3/assignment_loss‚0.2436596647441604 0loss/SRqqt1/detection_loss‚2.645922021071116 1loss/FBt1/assignment_loss‚0.41904255911059507 %loss/total_loss‚45.87436467138957 % _timestamp‚1.7884533119345138e+09 0loss/FRt1/assignment_loss‚3.6012694316032605 /loss/FRt1/detection_loss‚2.7497129073509803 _step‚871 1loss/SRqqt4/assignment_loss‚1.542879479787042 0loss/FBt4/assignment_loss‚0.3477600751866703 0loss/SRqqt3/detection_loss‚1.078723198389762 /loss/FBt2/detection_loss‚2.2703917175531387 /loss/FBt3/detection_loss‚0.9393990428852184 0loss/FRt3/assignment_loss‚1.8287640649538774 /loss/FRt3/detection_loss‚1.8046865372990186 1loss/SRqqt2/detection_loss‚1.9297436057608954 .loss/FBt4/detection_loss‚3.706367090344429 trainer/global_step‚10599 2loss/SRqqt3/assignment_loss‚1.1680677806677884 /loss/FBt1/detection_loss‚3.1960037549336753 /loss/FRt4/assignment_loss‚2.780461304462873 .loss/FRt2/detection_loss‚2.055902916295775 epoch‚8 0loss/FRt2/assignment_loss‚3.6177640916972327 2loss/SRqqt1/assignment_loss‚1.9058939119180043 2loss/SRqqt2/assignment_loss‚1.8698874762547868çä2‚*‘'‚(:† ÂÃæÔ°Åþü gpu.0.gpu‚38.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚171.82' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.364 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0žQx'‚(:í ÂÃæÔâÅÝÿ disk.drbd2266.out‚0 disk.sda1.out‚0# memory_percent‚94.1957950592041 cpu‚25.336419457403952 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875( disk.nvme0c0n1.out‚34095.26318359375 disk.sda2.in‚0 disk.sda2.out‚0 network.sent‚ 2.60281e+07 disk.nvme0n1.in‚0.078125 disk.sda3.out‚ 326.48046875 disk.drbd2266.in‚0 disk.sda1.in‚0' proc.memory.availableMB‚ 950.9609375 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125# disk.dm-0.out‚34095.26318359375) proc.memory.percent‚16.14522933959961 disk.rbd1.out‚ 48.498046875 disk.rbd0.out‚0.078125! proc.memory.rssMB‚ 2645.234375 proc.cpu.threads‚17* disk./.usagePercent‚20.378011303981953 disk.rbd1.in‚0& disk.nvme0n1.out‚34095.26318359375 disk.dm-1.out‚ 326.48046875 disk.drbd2265.out‚0 disk.drbd2080.in‚0 network.recv‚ 3.025627e+06$ disk./.usageGB‚303.6470947265625 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.sda.out‚ 326.48046875 disk.drbd1716.out‚0 disk.drbd2265.in‚0ë鲞‘'‚(:“ ÑÃæÔÎàîü gpu.0.gpu‚26.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚181.168' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.233599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0»hè±w’'‚(:ì ÑÃæÔÝùñÿ disk.dm-1.out‚ 327.0703125 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚34127.68896484375( proc.memory.availableMB‚ 950.96484375 network.recv‚ 3.030652e+06 disk.sda3.out‚ 327.0703125 disk.rbd0.out‚0.078125 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.drbd2266.in‚0 proc.cpu.threads‚17 disk.drbd2266.out‚0# disk.dm-0.out‚34127.68896484375 disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 48.560546875) proc.memory.percent‚16.14522933959961 disk.drbd2080.out‚0$ disk./.usageGB‚303.6606903076172 disk.rbd1.in‚0! proc.memory.rssMB‚ 2645.234375 network.sent‚ 2.613572e+07( disk.nvme0c0n1.out‚34127.68896484375 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda.out‚ 327.0703125 disk.sda1.in‚0* disk./.usagePercent‚20.378923714833984 disk.drbd2265.out‚0$ memory_percent‚94.19577121734619 cpu‚25.33588197475594 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.out‚0»)óù““'‚(:ˆ àÃæÔó²«ý gpu.0.gpu‚49.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚178.777' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.7554 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 F!wq”'‚(:æ àÃæԽ̙ÿ$ disk.nvme0n1.out‚34174.474609375 disk.drbd1716.in‚0 network.recv‚ 3.035708e+06 disk.drbd2266.in‚0 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2266.out‚0! disk.dm-0.out‚34174.474609375 disk.sda2.out‚0! proc.memory.rssMB‚ 2645.234375 cpu‚25.335381054968845 disk.drbd2265.out‚0 disk.sda.out‚ 327.7578125 disk.rbd1.in‚0 disk.rbd1.out‚ 48.560546875) proc.memory.percent‚16.14522933959961 disk.dm-1.out‚ 327.7578125 proc.cpu.threads‚17* disk./.usagePercent‚20.374366012701906 disk.sda3.out‚ 327.7578125 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.sda1.in‚0 disk.drbd2080.out‚0 disk.sda1.out‚0% disk./.usageGB‚303.59277725219727& disk.nvme0c0n1.out‚34174.474609375 disk.sda.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875$ memory_percent‚94.19574737548828% proc.memory.availableMB‚ 950.96875 network.sent‚ 2.6142193e+07 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125ÀÌ{f2•'- _wandbruntime‚8463  _runtime‚8463'é0jÊ–'Ä _step‚872 ' lr-AdamW/pg1‚5.4036101286924966e-06 ' lr-AdamW/pg2‚5.4036101286924966e-06 trainer/global_step‚10649 % _timestamp‚1.7884533485063975e+09 _runtime‚8463.978070369$¹ÚIÏ—'É ' lr-AdamW/pg1‚5.4036101286924966e-06 ' lr-AdamW/pg2‚5.4036101286924966e-06 trainer/global_step‚10649 % _timestamp‚1.7884533485063975e+09 _runtime‚8463.978070369 _step‚872èB872˜'- _wandbruntime‚8463  _runtime‚8463TU¦kç'™'áO M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 154724220 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 355663414 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 141176709 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚90272918 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8913885485056 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚90272916 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚675690357178880 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚627653652579840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚627652978214400 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 200959232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚602532728634368 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚50903352 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚685043805477376 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8914541847552 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8913885485056 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚439 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚10649 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚618739110732288 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 355683454 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 200939194 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚611413799837276 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚627652978214400 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 154724222 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚611414471314392 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 141176268 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚685043617913344 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8881725640664 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚627653652579840 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚50903791 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 200939194 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8914541847552 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8881071202908 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 _step‚873 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9353424358400 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚50903352 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚685043805477376 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8914541847552 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8913885485056 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚439 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 trainer/global_step‚10649 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚618739110732288 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 355683454 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 200939194 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚611413799837276 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚627652978214400 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 154724222 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚611414471314392 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 141176268 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚685043617913344 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8881725640664 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚627653652579840 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚50903791 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 200939194 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8914541847552 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8881071202908 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 _step‚873 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9353424358400 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128é|Ž-2›'- _wandbruntime‚8464  _runtime‚8464H8VGé œ'ìM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8882542075836 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚627710825059840 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8914704912896 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 141189445 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 154738236 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 141188991 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚611469520402800 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 355695884 Eo batch_end/inactive_split_bytes.large_pool.freed‚675752007916544 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 200977712 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9354275890688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚685106109710848 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8915361288704 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚90281086 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚8464.662213835 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8881887637872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚618795445768192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚50908359 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 355715950 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚675752031856640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8915361288704 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚685106307747328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚602587649804288 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 355695884 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 % _timestamp‚1.7884533491901968e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 200957648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9354101794304–(j/÷&'ñM OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 154738236 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 355715950 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8881887637872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚618795445768192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚50908359 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 355715950 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚675752031856640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8915361288704 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚685106307747328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚602587649804288 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 355695884 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 % _timestamp‚1.7884533491901968e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 200957648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9354101794304 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8882542075836 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚627710825059840 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8914704912896 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 141189445 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 154738236 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 141188991 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚611469520402800 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 355695884 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚685106109710848 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 8915361288704 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚90281086 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚8464.662213835 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 155425022 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8954871290880 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚621556762674688 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚678772918002176 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚688169335491072 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 201885284 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚630512290328064 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9396605052928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8922560980072 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚621556744671744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚630511615962624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚621556744671744 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8955527653376 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚51133291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 155425022 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 141814607 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚630511615962624 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8921906542316 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884533856694531e+09 _step‚877 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 201865246 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚90681316 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚605277935162368 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 155425020 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8955527653376 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8954871290880 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚90681318 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚470 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatvkGsMonitor.on_train_batch_start/active.all.freed‚ 357290266 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚621556762674688 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 155425020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 201885284 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚678772894062080 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872™""ïí'ª'çO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚688169335491072 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 201885284 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚630512290328064 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9396605052928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 8922560980072 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚621556744671744 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚630511615962624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚621556744671744 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8955527653376 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚51133291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 155425022 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 141814607 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚630511615962624 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 8921906542316 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7884533856694531e+09 _step‚877 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 201865246 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚90681316 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚605277935162368 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 155425020 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8955527653376 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8954871290880 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚90681318 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚470 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 357290266 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚621556762674688 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 155425020 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 201885284 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚678772894062080 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 155425022 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8954871290880 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚621556762674688 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚678772918002176 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928íç–R²2«'- _wandbruntime‚8501  _runtime‚8501—º€«ò&¬'ìM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚621613115713536 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚621613097710592 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚878 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚630568788429312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚630569462808064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚688232033692160 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚8501.840577345 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚621613097710592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 141827386 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚90689486 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚51137902 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚678834568739840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 357322736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 357342802 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 155439038 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 201883700 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 141827844 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 155439036 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 357322736 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8923377415244 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 155439038 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8956347094528 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚605332856332288 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 201883700 % _timestamp‚1.7884533863685768e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚630569462808064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚614255562270208 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 357342802 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚621613115713536 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8955690718720G·^÷&­'ñM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚458 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 201883700 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 141827844 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 155439036 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 357322736 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 8923377415244 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 155439038 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8956347094528 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚605332856332288 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 201883700 % _timestamp‚1.7884533863685768e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚630569462808064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚614255562270208 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 357342802 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚621613115713536 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8955690718720 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚621613115713536 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚621613097710592 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _step‚878 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚630568788429312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚630569462808064 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚688232033692160 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚8501.840577345 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚621613097710592 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 141827386 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚90689486 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚51137902 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚678834568739840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 357322736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 357342802 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 155439038 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0î{ùm2®'- _wandbruntime‚8501  _runtime‚8501Üœ$^^¯'Ø 2loss/SRqqt3/assignment_loss‚1.2697979621051882 0loss/FBt1/assignment_loss‚0.3590968817413265 .loss/FBt4/detection_loss‚3.242564252444676 _step‚879 2loss/SRqqt4/assignment_loss‚1.4581480716034525 /loss/FBt3/detection_loss‚0.7417035911662074 2loss/SRqqt1/assignment_loss‚2.3166454093796864 2loss/SRqqt2/assignment_loss‚1.4240500804077347 /loss/FRt4/detection_loss‚1.9719532343649095 1loss/SRqqt2/detection_loss‚1.9115859639385473 1loss/SRqqt4/detection_loss‚1.4963257110021153 trainer/global_step‚10699 0loss/FRt4/assignment_loss‚3.1810879938064085 .loss/FRt2/detection_loss‚2.066528924201664 /loss/FBt2/detection_loss‚3.3115307688713074 0loss/FRt3/assignment_loss‚1.6935614563267807 epoch‚8 % _timestamp‚1.7884533863701456e+09 /loss/FRt1/assignment_loss‚4.225621467544919 1loss/FBt3/assignment_loss‚0.19193934615118335 0loss/FBt4/assignment_loss‚0.1383520984756095 &loss/total_loss‚47.389438554188175 0loss/FRt2/assignment_loss‚3.6498416756328784 /loss/FBt2/assignment_loss‚0.477155695437735 1loss/SRqqt1/detection_loss‚2.7091668993234634 1loss/SRqqt3/detection_loss‚1.3882659835597644 /loss/FRt1/detection_loss‚3.7199850366229104 /loss/FRt3/detection_loss‚1.8615534267035023 /loss/FBt1/detection_loss‚2.5829766233762106 _runtime‚8501.841988107ÛZKxé°'Ý 0loss/FRt2/assignment_loss‚3.6498416756328784 /loss/FBt2/assignment_loss‚0.477155695437735 1loss/SRqqt1/detection_loss‚2.7091668993234634 1loss/SRqqt3/detection_loss‚1.3882659835597644 /loss/FRt1/detection_lophSùzss‚3.7199850366229104 /loss/FRt3/detection_loss‚1.8615534267035023 /loss/FBt1/detection_loss‚2.5829766233762106 _runtime‚8501.841988107 2loss/SRqqt3/assignment_loss‚1.2697979621051882 0loss/FBt1/assignment_loss‚0.3590968817413265 .loss/FBt4/detection_loss‚3.242564252444676 _step‚879 2loss/SRqqt4/assignment_loss‚1.4581480716034525 /loss/FBt3/detection_loss‚0.7417035911662074 2loss/SRqqt1/assignment_loss‚2.3166454093796864 2loss/SRqqt2/assignment_loss‚1.4240500804077347 /loss/FRt4/detection_loss‚1.9719532343649095 1loss/SRqqt2/detection_loss‚1.9115859639385473 1loss/SRqqt4/detection_loss‚1.4963257110021153 trainer/global_step‚10699 0loss/FRt4/assignment_loss‚3.1810879938064085 .loss/FRt2/detection_loss‚2.066528924201664 /loss/FBt2/detection_loss‚3.3115307688713074 0loss/FRt3/assignment_loss‚1.6935614563267807 epoch‚8 % _timestamp‚1.7884533863701456e+09 /loss/FRt1/assignment_loss‚4.225621467544919 1loss/FBt3/assignment_loss‚0.19193934615118335 0loss/FBt4/assignment_loss‚0.1383520984756095 &loss/total_loss‚47.389438554188175ï7%‡“±'‚(:ˆ ÄæÔÍ­éý gpu.0.gpu‚24.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.368' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.2736 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ãÿ—ÿ{²'‚(:ð ÄæÔîÔÎÿ( disk.nvme0c0n1.out‚34322.78564453125 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.sda.out‚ 329.62890625 disk.rbd1.out‚ 49.052734375 proc.cpu.threads‚17 network.sent‚ 2.6262016e+07) disk./.usagePercent‚20.37804919309702$ disk./.usageGB‚303.6476593017578 disk.sda1.in‚0 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 network.recv‚ 3.050048e+06 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚34322.78564453125 disk.drbd1716.in‚0( proc.memory.availableMB‚ 950.87109375! proc.memory.rssMB‚ 2645.234375 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 329.62890625 disk.sda2.out‚0 disk.dm-1.out‚ 329.62890625 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0) proc.memory.percent‚16.14522933959961 disk.rbd0.in‚0 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2266.in‚0$ memory_percent‚94.19634342193604 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 cpu‚25.333844408645298# disk.dm-0.out‚34322.78564453125 disk.drbd2080.out‚0`+Š“³'‚(:ˆ œÄæÔÄõøü gpu.0.gpu‚46.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚186.764' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.3528 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0è²n´'‚(:ã œÄæÔ„†€ disk.rbd0.in‚0 network.recv‚ 3.056892e+06 disk.rbd1.in‚0 disk.rbd1.out‚ 49.619140625 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.dm-0.in‚0.078125* disk./.usagePercent‚20.378579640707965 disk.sda1.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2266.out‚0! disk.dm-0.out‚34375.380859375$ disk.nvme0n1.out‚34375.380859375! proc.memory.rssMB‚ 2645.234375 disk.drbd2266.in‚0 disk.sda2.out‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0$ memory_percent‚94.19634342193604 network.sent‚ 2.6370869e+07 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0.078125 disk.sda.out‚ 329.984375( proc.memory.availableMB‚ 950.87109375) proc.memory.percent‚16.14522933959961 cpu‚25.3333426820969 disk.dm-1.out‚ 329.984375$ disk./.usageGB‚303.6555633544922 disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0& disk.nvme0c0n1.out‚34375.380859375 disk.sda3.in‚ 0.01171875 proc.cpu.threads‚17 disk.sda3.out‚ 329.984375²åü9žµ'‚(:“ «ÄæÔØì€ý gpu.0.gpu‚46.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚175.696' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.139199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÑÊ:´h¶'‚(:Ý «ÄæÔÍÖ¢ÿ disk.nvme0n1.in‚0.078125 disk.dm-1.out‚330.6875$ memory_percent‚94.19634342193604 network.recv‚ 3.059986e+06) disk./.usagePercent‚20.37927828526891 disk.rbd1.in‚0 disk.rbd1.out‚ 50.501953125 disk.nvme0c0n1.in‚0.078125& disk.nvme0c0n1.out‚34417.826171875 disk.sda3.out‚330.6875! proc.memory.rssMB‚ 2645.234375 disk.dm-1.in‚ 0.01171875 disk.dm-0.in‚0.078125( proc.memory.availableMB‚ 950.87109375 cpu‚25.332852262339966 disk.drbd2265.in‚0 disk.sda1.out‚0 network.sent‚ 2.637602e+07 disk.sda1.in‚0) proc.memory.percent‚16.14522933959961$ disk.nvme0n1.out‚34417.826171875 disk.drbd1716.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125! disk.dm-0.out‚34417.826171875 disk.sda.out‚330.6875 disk.sda2.out‚0$ disk./.usageGB‚303.6659736633301 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda.in‚ 0.01171875 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda2.in‚0{ª 2·'- _wandbruntime‚8538  _runtime‚8538 ‡\ ȸ' trainer/global_step‚10749 % _timestamp‚1.7884534228234355e+09 _runtime‚8538.295026853 _step‚880 & lr-AdamW/pg1‚4.688729505837013e-06 & lr-AdamW/pg2‚4.688729505837013e-06~¿_¼Í¹'Ç _step‚880 & lr-AdamW/pg1‚4.688729505837013e-06 & lr-AdamW/pg2‚4.688729505837013e-06 trainer/global_step‚10749 % _timestamp‚1.7884534228234355e+09 _runtime‚8538.295026853ðùáæt2º'- _wandbruntime‚8538  _runtime‚8538¿¦Åmè'»'âO UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 202811336 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 358937158 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 8996513459200 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚624374396614144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚8538.296159411 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 156125822 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8995857096704 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 142452545 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 202791298 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8996513459200 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8995857096704 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _step‚881 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚616986555049208 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 202811336 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚633370253710848 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚91089718 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚633370928076288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚681855430945280 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 156125822 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9439694280704 % _timestamp‚1.7884534228242488e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚633370253710848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 156125820 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 142452988 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚691294963699200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚608023141690368 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9439532753920 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 _step‚881 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚616986555049208 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 202811336 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚633370253710848 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚91089718 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚633370928076288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚681855430945280 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 156125822 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9439694280704 % _timestamp‚1.7884534228242488e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚633370253710848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 156125820 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 142452988 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚691294963699200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚608023141690368 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9439532753920 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _runtime‚8538.296159411 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 156125822 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 8995857096704 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 142452545 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 202791298 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 8996513459200 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 8995857096704 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚617042275614940 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 202829816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 156139836 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚691357470340608 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 _step‚882 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 202829816 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 142465236 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8997332900352 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚617041604137616 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 358969654 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8963558316688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8996676524544 H:DeviceStatsM¸½:bêonitor.on_train_batch_end/allocation.all.freed‚ 358949588 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚681917081682944 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚624430767655936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚633427426177536 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚51367800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9440388657664 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884534237860227e+09 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 358969654 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 202809752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 358949588 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚633428100556288 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚10749 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚91097884 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8996676524544 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚51367352uK÷&¿'ñM JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚608078045820928 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚624430749652992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 156139838 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 142465686 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚681917105623040 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚8539.258337562 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚624430767655936 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 358969654 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 8963558316688 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 8996676524544 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 358949588 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚681917081682944 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚624430767655936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚633427426177536 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚51367800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9440388657664 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884534237860227e+09 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 358969654 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 202809752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 358949588 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚633428100556288 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚10749 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚91097884 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 8996676524544 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚51367352 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9440554365440 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚608078062860288 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚617042275614940 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 202829816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 156139836 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚691357470340608 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 _step‚882 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 202829816 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 142465236 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 8997332900352 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚617041604137616 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚636228891459072 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚636228891459072 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 143091015 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 156826622 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚636229565824512 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 156826622 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚51592897 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚51592423 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚694420806600192 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚619771925439500 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚627192066559488 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚627192048556544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚91498118 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚476 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚474 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 156826620S _?ãÌ'çO RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 143091015 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 156826622 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚636229565824512 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 156826622 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚51592897 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚51592423 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚694420806600192 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚619771925439500 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚627192066559488 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚627192048556544 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚91498118 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚476 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚474 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 156826620 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚684937967828480 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚885 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚636229565824512 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9036842902528 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 trainer/global_step‚10799 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚627192048556544 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 143090539 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 183369728 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 360564010 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9003577221132 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 203717350 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9036842902528 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚636228891459072 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚636228891459072õ,ž„›2Í'- _wandbruntime‚8576  _runtime‚8576dëÁ"ñ&Î'ëM ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 156840638 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 360596506 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 $ _timestamp‚1.788453461218431e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚627248419598336 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚636286738304512 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚684999642506240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚610823252348928 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚627248401595392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚694483317710336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9483675204096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚619828317482348 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9004393656096 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚627248419598336 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9037662330368 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _step‚886 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚694483125965312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚636286063925760 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚619827646005024 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 360596506 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 360576440 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚8576.690635424 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 156840636 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 143103748 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚51597462 trainer/global_step‚10799 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 143103265 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 203755868 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚636286738304512 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚627248401595392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9038318706176 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚481 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9483507399168 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 203735804 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9037662330368 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 156840638 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 360576440 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚483 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PRö&Ï'ðM OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚481 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9483507399168 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 203735804 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9037662330368 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 156840638 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 360576440 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚483 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 156840638 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 360596506 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 $ _timestamp‚1.788453461218431e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚627248419598336 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚636286738304512 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚684999642506240 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚610823252348928 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚627248401595392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚694483317710336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9483675204096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚619828317482348 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9004393656096 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚627248419598336 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9037662330368 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 _step‚886 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚694483125965312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚636286063925760 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚619827646005024 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 360596506 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 360576440 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚8576.690635424 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 156840636 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 143103748 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚51597462 trainer/global_step‚10799 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 143103265 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 203755868 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚636286738304512 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚627248401595392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9038318706176öÓ¶w}2Ð'- _wandbruntime‚8576  _runtime‚8576ËAk¯XÑ'Ò 2loss/SRqqt1/assignment_loss‚1.5756722300623853 2loss/SRqqt2/assignment_loss‚1.5815571859155013 0loss/FBt2/assignment_loss‚0.3057611112793186 /loss/FBt4/assignment_loss‚0.641987645377713 0loss/SRqqt2/detection_loss‚1.924298820288285 /loss/FBt1/detection_loss‚3.0017247200012207 /loss/FBt4/detection_loss‚3.6190743446350098 1loss/FBt3/assignment_loss‚0.21325634162598045 /loss/FBt2/detection_loss‚1.9978099465370178 trainer/global_step‚10799 0loss/FRt3/assignment_loss‚1.8389875908692679 1loss/SRqqt1/detection_loss‚2.9614610274632773 /loss/FBt3/detection_loss‚0.7093669993548908 /loss/FRt4/assignment_loss‚2.256234248876572 2loss/SRqqt4/assignment_loss‚1.3423857723163175 1loss/SRqqt4/detection_loss‚1.7002145731821656 $ _timestamp‚1.788453461220183e+09 _runtime‚8576.691708379 epoch‚8 /loss/FRt1/assignment_loss‚3.692705351373424 /loss/FRt4/detection_loss‚2.0345297239720823 $loss/total_loss‚44.5010623938859 /loss/FRt2/assignment_loss‚3.451495399308759 0loss/FBt1/assignment_loss‚0.4430552557867158 /loss/FRt3/detection_loss‚1.5560674014190832 0loss/SRqqt3/assignment_loss‚1.26448220562722 /loss/FRt1/detection_loss‚2.9331144871919053 /loss/FRt2/detection_loss‚2.4040771244570265 1loss/SRqqt3/detection_loss‚1.0517428869647638 _step‚887F M]Ò'× /loss/FRt1/assignment_loss‚3.692705351373424 /loss/FRt4/detection_loss‚2.0345297239720823 $loss/total_loss‚44.5010623938859 /loss/FRt2/assignment_loss‚3.451495399308759 0loss/FBt1/assignment_loss‚0.4430552557867158 /loss/FRt3/detection_loss‚1.5560674014190832 0loss/SRqqt3/assignment_loss‚1.26448220562722 /loss/FRt1/detection_loss‚2.9331144871919053 /loss/FRt2/detection_loss‚2.4040771244570265 1loss/SRqqt3/detection_loss‚1.0517428869647638 _step‚887 2loss/SRqqt1/assignment_loss‚1.5756722300623853 2loss/SRqqt2/assignment_loss‚1.5815571859155013 0loss/FBt2/assignment_loss‚0.3057611112793186 /loss/FBt4/assignment_loss‚0.641987645377713 0loss/SRqqt2/detection_loss‚1.924298820288285 /loss/FBt1/detection_loss‚3.0017247200012207 /loss/FBt4/detection_loss‚3.6190743446350098 1loss/FBt3/assignment_loss‚0.21325634162598045 /loss/FBt2/detection_loss‚1.9978099465370178 trainer/global_step‚10799 0loss/FRt3/assignment_loss‚1.8389875908692679 1loss/SRqqt1/detection_loss‚2.9614610274632773 /loss/FBt3/detection_loss‚0.7093669993548908 /loss/FRt4/assignment_loss‚2.256234248876572 2loss/SRqqt4/assignment_loss‚1.3423857723163175 1loss/SRqqt4/detection_loss‚1.7002145731821656 $ _timestamp‚1.788453461220183e+09 _runtime‚8576.691708379 epoch‚8÷´}Á9žÓ'‚(:“ ØÄæÔ±¯Éþ gpu.0.gpu‚39.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.168' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.233599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Aó§õ{Ô'‚(:ð ØÄæÔŽ¿Èÿ disk.sda1.out‚0 network.recv‚ 3.075828e+06& disk.nvme0n1.out‚34560.75537109375 disk.dm-1.out‚ 332.27734375 disk.nvme0c0n1.in‚0.078125 disk.sda2.in‚0% disk./.usageGB‚303.69491958618164 disk.drbd1716.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.sda.out‚ 332.27734375$ memory_percent‚94.19827461242676# disk.dm-0.out‚34560.75537109375* disk./.usagePercent‚20.381220870438725 disk.nvme0n1.in‚0.078125( disk.nvme0c0n1.out‚34560.75537109375 disk.rbd0.out‚0.078125 cpu‚25.33134291361042 network.sent‚ 2.6496942e+07 disk.drbd2080.out‚0 disk.rbd0.in‚0' proc.memory.availableMB‚ 950.5546875! proc.memory.rssMB‚ 2645.234375 disk.drbd2265.out‚0 disk.sda3.out‚ 332.27734375 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 50.580078125) proc.memory.percent‚16.14522933959961 disk.sda1.in‚0 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 proc.cpu.threads‚17þH“8—Õ'‚(:ˆ çÄæÔ’º•ý gpu.0.gpu‚53.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 1074167¥j9Hü8080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.699' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.3398 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0w+ärwÖ'‚(:ì çÄæÔת¡ÿ disk.drbd2266.out‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 network.sent‚ 2.6604488e+07 network.recv‚ 3.081018e+06 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.drbd2265.out‚0! proc.memory.rssMB‚ 2645.234375 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.rbd1.out‚ 50.8076171875 disk.dm-1.in‚ 0.01171875$ memory_percent‚94.19674873352051 proc.cpu.threads‚17* disk./.usagePercent‚20.382978362161282 disk.rbd0.out‚0.078125' proc.memory.availableMB‚ 950.8046875 disk.sda.out‚ 332.9453125 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 332.9453125" disk.dm-0.out‚34611.1767578125 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 332.9453125% disk./.usageGB‚303.72110748291016 disk.sda1.in‚0% disk.nvme0n1.out‚34611.1767578125 disk.drbd2265.in‚0 disk.drbd2080.out‚0) proc.memory.percent‚16.14522933959961 cpu‚25.330826477481967' disk.nvme0c0n1.out‚34611.1767578125 disk.sda3.in‚ 0.01171875çÍšmž×'‚(:“ öÄæÔ¯©”ý gpu.0.gpu‚47.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚188.731' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.746199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‚ÂuØ'‚(:ê öÄæÔ»Ÿ¯€* disk./.usagePercent‚20.384809840061237 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0 proc.cpu.threads‚17 disk.dm-1.in‚ 0.01171875 disk.dm-0.in‚0.078125! proc.memory.rssMB‚ 2645.234375) proc.memory.percent‚16.14522933959961 cpu‚25.330332183406473 disk.sda1.out‚0 disk.rbd1.out‚ 50.8076171875 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚34665.2666015625 disk.sda3.in‚ 0.01171875 disk.drbd2266.out‚0$ memory_percent‚94.19844150543213% disk.nvme0n1.out‚34665.2666015625 disk.drbd2080.in‚0 disk.sda3.out‚ 333.609375 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0.078125" disk.dm-0.out‚34665.2666015625( proc.memory.availableMB‚ 950.52734375 network.recv‚ 3.086419e+06 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 333.609375 disk.drbd2265.in‚0 disk.sda.out‚ 333.609375 network.sent‚ 2.6611252e+07 disk.drbd1716.out‚0 disk.drbd2080.out‚0% disk./.usageGB‚303.74839782714844aq“2Ù'- _wandbruntime‚8613  _runtime‚8613–D«JÈÚ' % _timestamp‚1.7884534977917466e+09 _runtime‚8613.263338549 _step‚888 & lr-AdamW/pg1‚4.022740055809757e-06 & lr-AdamW/pg2‚4.022740055809757e-06 trainer/global_step‚10849"ã'ÍÛ'Ç & lr-AdamW/pg1‚4.022740055809757e-06 & lr-AdamW/pg2‚4.022740055809757e-06 trainer/global_step‚10849 % _timestamp‚1.7884534977917466e+09 _runtime‚8613.263338549 _step‚888ø‹øÚL2Ü'- _wandbruntime‚8613  _runtime‚8613:ƒÌè'Ý'âO UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 143729343 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9045066998296 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 204663440 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚436 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚434 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚613513571785728 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9078485070848 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚630009700498944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9078485070848 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚639087529207296 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 157527420 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 168689664 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚697546208899072 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9044412560540 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 362190862 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚51822391 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚639088203572736 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚622558638784024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 204643402 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚91906518 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚613513554746368 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚91906516 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚697546377588736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚630009700498944 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚688020504711680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚639088203572736 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚688020528651776 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 % _timestamp‚1.7884534977924318e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 204643402 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚630009718501888 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9525848936960 _runtime‚8613.264262252 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 157527420 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9077828708352 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 157527422 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9525704187392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚630009718501888 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚51822825 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 204663440 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚889 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 157527422 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9077828708352 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 143728907 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚639087529207296 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 157527420 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 168689664 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚697546208899072 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9044412560540 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 362190862 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚51822391 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚639088203572736 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚622558638784024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 204643402 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚91906518 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚613513554746368 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚91906516 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚697546377588736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚630009700498944 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚688020504711680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚639088203572736 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚688020528651776 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 % _timestamp‚1.7884534977924318e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 204643402 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚630009718501888 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9525848936960 _runtime‚8613.264262252 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 157527420 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9077828708352 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 157527422 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9525704187392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚630009718501888 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚51822825 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 204663440 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚889 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 157527422 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9077828708352 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 143728907 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 362223358 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884534984987824e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9078648136192 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚697608904598528 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9045228995504 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚8613.970995872 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 143742199 trainer/global_step‚10849 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 362223358 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 157541438 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 204681920 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 204661856 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚470 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 143741729 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚697608719144960 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚630066071540736 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚630066053537792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 362203292 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚688082155449344 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚613568458876928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 157541438 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 157541436 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 204661856 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚630066071540736 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 W2äÌIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 204681920 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚639145376052736 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚639144701673984 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9526725209088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚622614359349756 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚622613687872432 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚630066053537792 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚51827045 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 362223358 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 % _timestamp‚1.7884534984987824e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9078648136192 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚697608904598528 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9045228995504 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚8613.970995872 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 143742199 trainer/global_step‚10849 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 362223358 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 157541438 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 204681920 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 204661856 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚470 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 143741729 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚697608719144960 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚630066071540736 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚630066053537792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 362203292 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚688082155449344 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚613568458876928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 157541438 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 157541436 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 204661856 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚630066071540736 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 204681920 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚639145376052736 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚639144701673984 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9526725209088 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚622614359349756 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚622613687872432 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚630066053537792 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚51827045 BDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 \JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚ 9122126087168 QGDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak‚24128 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 % _timestamp‚1.7884535701964898e+09 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 WIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚ 144682410 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚626088929947188Έg#+ñ'ÎT KEDeviceStatsMonitor.on_validation_ š5\)*batch_start/segment.small_pool.freed‚0 trainer/global_step‚10880 _step‚892 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 132992512 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 _MDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.freed‚ 9122126087168 VHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚ 158505426 hVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚ 9571637531648 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚ 158505428 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 205612507 jVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.allocated‚691849263198720 B;DeviceStatsMonitor.on_validation_batch_start/max_split_size‚-1 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 [LDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.peak‚ 8838745088 [GDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.allocated‚642740306073600 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.freed‚ 144682253 cODeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.allocated‚701420900730368 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚626088421993436 cQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.allocated‚ 9122618917376 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.allocated‚92486590 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 ]ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.current‚ 490914392 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 _MDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.freed‚ 9088540854236 XIDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.peak‚ 9148669952 ]ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.current‚ 492830208 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 VNDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.current‚155 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 QIDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.allocated‚414 `NDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚ 9122618917376 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚691849239258624 \NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.current‚ 868220928 _KDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.freed‚701420767737856 E=DeviceStatsMonitor.on_validation_batch_start/num_device_alloc‚844 OIDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 364117935 RKDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.peak‚30 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚52195820 XJDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.peak‚ 339635712 dRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.freed‚ 9571528479232 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚616999881139200 bNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚633617687156224 ^JDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.allocated‚642740306073600 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 FDeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 LFDeviceStatsMonitor.on_validation_batch_start/active.large_pool.current‚2 ZLDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.current‚ 492830208 \JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚ 9122126087168 QGDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.peak‚24128 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 ;5DeviceStatsMonitor.on_validation_batch_start/num_ooms‚0 % _timestamp‚1.7884535701964898e+09 MCDeviceStatsMonitor.on_validation_batch_start/active.small_pool.peak‚24128 WIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚ 144682410 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚626088929947188 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚642739795240448 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚642739795240448 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510833152 TNDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.current‚2 eQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.allocated‚616999898178560 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 _runtime‚8685.668456431 ^JDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.freed‚633617669153280 cQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚ 9089031768628 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 PFDeviceStatsMonitor.on_validation_batch_start/active.small_pool.current‚15977 RDDeviceStatsMonitor.on_validation_batch_start/active.small_pool.freed‚ 205596530 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 I?DeviceStatsMonitor.on_validation_batch_start/active.all.current‚15979 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 bTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.current‚ 109052416 K=DeviceStatsMonitor.on_validation_batch_start/active.all.freed‚ 364101956 VHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚ 158505428 RDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚ 158505426 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.current‚ 510833152 TEDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_validation_batch_start/allocation.all.allocated‚ 364117935 WIDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.peak‚ 863815680 aMDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.freed‚633617669153280 OADeviceStatsMonitor.on_validation_batch_start/allocation.all.freed‚ 364101956 QKDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_validation_batch_start/segment.all.allocated‚844 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.peak‚414 QBDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.peak‚ 9955586048 eQDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.allocated‚633617687156224 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚92486588 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚52195665 WGDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.current‚ 10194255872 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 VHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚ 205612507 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚633654517185024 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 9572002658304 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 OIDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚ 205619809 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 QIDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.peak‚889 I;DeviceStatsMonitor.on_validation_batch_end/active.all.freed‚ 364120064 UADeviceStatsMonitor.on_validation_batch_end/active_bytes.all.freed‚642776990395904 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 9122491213824 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚633654499182080 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 bPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚ 9571893605888 [LDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.current‚ 9326034944 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚633654499182080 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680 ]KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚ 9122491213824 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 364136043 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 hTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚691887138932224 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 $ _timestamp‚1.788453571152092e+09 ^LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚ 9122984044032 M?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚ 364120064 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 _runtime‚8686.624133905 OEDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.peak‚24128 MEDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.current‚157 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 109052416 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 RCDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.peak‚ 9643699025 KADeviceStatsMonitor.on_validation_batch_end/active.small_pool.peak‚24128 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚92493092 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚155 aMDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.allocated‚701459141590528 trainer/global_step‚10880 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 364136043`zþQ])ô'×R ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚ 144693566 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 XDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚642776990395904 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 cODeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.allocated‚633654517185024 _RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.current‚23940096 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚626124587316084 ICDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚ 158516234 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.allocated‚ 205619809 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 339635712 VGDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak‚ 9148669952 YJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 OGDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.allocated‚414 `LDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.allocated‚633654517185024 fTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚ 9572002658304 C;DeviceStatsMonitor.on_validation_batch_end/num_device_alloc‚844 OIDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_validation_batch_end/active.small_pool.allocated‚ 205619809 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 QIDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.peak‚889 I;DeviceStatsMonitor.on_validation_batch_end/active.all.freed‚ 364120064 UADeviceStatsMonitor.on_validation_batch_end/active_bytes.all.freed‚642776990395904 ZHDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.freed‚ 9122491213824 KADeviceStatsMonitor.on_validation_batch_end/allocation.all.current‚15979 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚633654499182080 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 bPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚ 9571893605888 [LDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.current‚ 9326034944 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚633654499182080 UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680 ]KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚ 9122491213824 QCDeviceStatsMonitor.on_validation_batch_end/allocation.all.allocated‚ 364136043 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 JADeviceStatsMonitor.on_validation_batch_end/active.large_pool.peak‚2605 hTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.allocated‚691887138932224 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 $ _timestamp‚1.788453571152092e+09 ^LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚ 9122984044032 M?DeviceStatsMonitor.on_validation_batch_end/allocation.all.freed‚ 364120064 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 @:DeviceStatsMonitor.on_validation_batch_end/num_device_free‚0 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 _runtime‚8686.624133905 OEDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.peak‚24128 MEDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.current‚157 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 109052416 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 RCDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.peak‚ 9643699025 KADeviceStatsMonitor.on_validation_batch_end/active.small_pool.peak‚24128 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚92493092 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚155 aMDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.allocated‚701459141590528 trainer/global_step‚10880 M?DeviceStatsMonitor.on_validation_batch_end/active.all.allocated‚ 364136043 QCDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.current‚ 510833152 QCDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.freed‚ 144693409 WIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.peak‚ 868220928 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚642777501229056 JBDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.peak‚908 WJDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.freed‚52200317 ]IDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.freed‚701459008598016 MGDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.current‚0 ]NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 WGDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.allocated‚ 10194255872 TFDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.current‚ 510833152 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844ýc΀Ò2õ'- _wandbruntime‚8699  _runtime‚8699èŠÌ”ö'¦ @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.0963529720902443 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 7 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.1874316781759262 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.14902858436107635 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0972222313284874 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.347023785ÔÍh•ù11428833 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.4375 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.07926861196756363 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9659090638160706 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 'particle/accuracy‚0.850983738899231 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8124949932098389 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ,"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.625 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9886252284049988 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9925675988197327 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7424241900444031 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 jet/accuracy_6_of_6‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚1 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.980235755443573 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9881795644760132 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 :"Purity/1FRt2SRqqt0FBt/event_purity‚0.00266400258988142 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.014328940771520138 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9937499761581421 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 +"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.25 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.24342657625675201 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 8"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.569968581199646 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13044582307338715 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.6315789222717285 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.6764705777168274 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5678910613059998 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.15804597735404968 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.42653509974479675 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.20296765863895416 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 9"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.4656223654747009 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 9"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.4690631926059723 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3565123677253723 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.40753257274627686 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.40740740299224854 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.33518514037132263 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 ;"Purity/1FRt2SRqqt1FBt/event_purity‚0.003448275849223137 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.43378376960754395 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.6034639477729797 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.15909090638160706 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 jet/accuracy_5_of_7‚0 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 jet/accuracy_5_of_6‚0 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9267138838768005 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9444443583488464 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5846656560897827 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24497434496879578 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.0357520766556263 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.08452380448579788 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9744047522544861 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5705227255821228 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 & Purity/*FRt0SRqqt4FBt/FRt_purity‚1 ) Purity/4FRt1SRqqt0FBt/FRt_purity‚0.25 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9683907628059387 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9936092495918274 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 ) Purity/*FRt1SRqqt3FBt/FRt_purity‚0.25 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5602841973304749 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.3140290677547455 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5921592712402344 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.012946957722306252 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9464285969734192 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 *loss/val_total_loss‚44.986293942523176 5 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.96648108959198 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5651731491088867 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.18831966817378998 particle/accuracy_1_of_6‚1 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5835162401199341 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.4735683798789978 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.2703857719898224 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 *jet/accuracy_1_of_4‚0.9284613132476807 particle/accuracy_2_of_5‚1 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.5024924874305725 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.14077557623386383 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.10905349999666214 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 particle/accuracy_6_of_7‚1 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 epoch‚8 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08057189732789993 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9864341616630554 *jet/accuracy_1_of_3‚0.8874834179878235 particle/accuracy_2_of_2‚1 +"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.25 particle/accuracy_4_of_6‚1 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 jet/accuracy_5_of_5‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5678910613059998 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.14749863743782043 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.02017568051815033 +jet/accuracy_3_of_4‚0.05288912355899811 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.4000000059604645 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5381368398666382 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.35960084199905396 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.27616289258003235 :"Purity/0FRt1SRqqt2FBt/event_purity‚0.01867816224694252 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6445052623748779 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.2149708867073059 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5508672595024109 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 +particle/sensitivity‚0.3283292353153229 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 jet/accuracy_1_of_7‚1 jet/accuracy_3_of_7‚0 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 8"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.517579197883606 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_5‚0.9688428640365601 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3318150043487549 *jet/accuracy_2_of_5‚0.5087172985076904 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2916666567325592 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_2_of_3‚1 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.7708333134651184 % _timestamp‚1.7884535843322513e+09 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9896711111068726 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 particle/accuracy_5_of_6‚1 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.17557041347026825 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9533333778381348 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_6‚1 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.6234567165374756 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.47331154346466064 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ) Purity/4FRt1SRqqt*FBt/FRt_purity‚0.25 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.012373506091535091 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_1_of_1‚1 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.545882523059845 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.17499502003192902 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_6‚1 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.2418861985206604 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11062941700220108 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.38296133279800415 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.12260536849498749 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_3_of_4‚1 particle/accuracy_5_of_5‚1 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.37952542304992676 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 ,jet/accuracy_3_of_3‚0.019731007516384125 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.45643970370292664 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5082651972770691 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.2969917356967926 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9957951307296753 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.13026930391788483 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 6validation_average_jet_accuracy‚0.4523635804653168 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5783928036689758 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.13111525774002075 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 6 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.262037068605423 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 _runtime‚8699.806343938 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9913515448570251 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5922619104385376 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 +jet/accuracy_2_of_2‚0.10539604723453522 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.6466667056083679 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 particle/accuracy_5_of_7‚1 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9872517585754395 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.44476747512817383 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 (jet/accuracy_1_of_1‚0.51680988073349 ,jet/accuracy_4_of_5‚0.004096262622624636 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.6226851940155029 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 trainer/global_step‚10880 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.0625 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5955635905265808 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9859274625778198 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 ,"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.375 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4943965673446655 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.3927689790725708 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2046070098876953 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.03546099364757538 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2490079551935196 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.998068630695343 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1776694804430008 8"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.566490888595581 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 _step‚894 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.12719298899173737 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.14139007031917572 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5701664090156555 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.29793471097946167 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5773777961730957 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.7708333134651184 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9914966225624084 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 ;"Purity/2FRt0SRqqt2FBt/event_purity‚0.015350878238677979 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 ) Purity/1FRt1SRqqt3FBt/FRt_purity‚0.25 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1480751931667328 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17811717092990875 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.08456198871135712 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.13805745542049408 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.13282422721385956 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0972222313284874 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 jet/accuracy_4_of_7‚0 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.10115797072649002 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4833333492279053 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.9122807383537292 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9479167461395264 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6809585094451904 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.03912683576345444 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5545225739479065 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5541329383850098 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9628472924232483 ,jet/accuracy_4_of_4‚0.002807604381814599 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5699907541275024 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9809297919273376 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5672531127929688 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9867123961448669 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.0555555559694767 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt2FBt/event_purity‚0.01344086043536663 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.5325524210929871 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_5‚1 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5400616526603699 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5126854777336121 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 particle/accuracy_1_of_4‚1 6 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.420138955116272 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 8"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.606178343296051 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 *jet/accuracy_2_of_6‚0.3589743971824646 +jet/accuracy_3_of_6‚0.17307692766189575 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.44333332777023315 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5901129841804504 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 * Purity/4FRt*SRqqt*FBt/FRt_purity‚0.125 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.034150224179029465 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5785714387893677 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 particle/accuracy_4_of_7‚1 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.45735853910446167 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.0941978245973587 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.025299709290266037 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.25540032982826233 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.4431818127632141 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.2380952537059784 jet/accuracy_6_of_7‚0 particle/accuracy_1_of_7‚1 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4851851761341095 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7121211886405945 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 9"Purity/0FRt0SRqqt3FBt/event_purity‚0.0416666679084301 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13611596822738647 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.18232321739196777 +jet/accuracy_3_of_5‚0.08856885880231857 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5325080156326294 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09128113836050034 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9765334129333496 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ;"Purity/0FRt*SRqqt3FBt/event_purity‚0.035087719559669495 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5555999875068665 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9849824905395508 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.13705334067344666 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.18981482088565826 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5357142686843872 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 7 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.1533595621585846 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_7‚1 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 jet/accuracy_1_of_6‚1 jet/accuracy_2_of_7‚0.5 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.6388888955116272 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5887032747268677 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0030595569405704737 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 *jet/accuracy_2_of_4‚0.3569861948490143 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14148594439029694 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 *particle/specificity‚0.947686493396759 particle/accuracy_3_of_7‚1 particle/accuracy_7_of_7‚1 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3566661775112152 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 particle/accuracy_1_of_3‚1 particle/accuracy_3_of_3‚1 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6154837012290955 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.6034379601478577 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.13333334028720856 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.20955400168895721 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.25846946239471436 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.15214082598686218 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.25074276328086853 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5598620176315308 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5548207759857178 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1110139936208725 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.45016834139823914 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6277263760566711 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.34375 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9968725442886353 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5313672423362732 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9907218813896179 ;"Purity/2FRt*SRqqt2FBt/event_purity‚0.013565892353653908 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6350575089454651 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5669006109237671 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4270833134651184 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 8 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.41535383462905884 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.3645833432674408 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_3‚0.24576181173324585 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.28074711561203003 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.19304554164409637 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 @V%T,—&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9836171269416809 + Purity/1FRt0SRqqt3FBt/FRt_purity‚0.6875 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.6097221970558167 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4000000059604645 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_2‚0.7739431262016296 jet/accuracy_4_of_6‚0 'particle/f_score‚0.4074062705039978 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.18588514626026154 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 6 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.988304078578949 7 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.1896396428346634 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.1779792457818985 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3263888955116272 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9804672002792358 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9896175265312195 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.03425183519721031 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5917125344276428 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3214285671710968 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5638038516044617 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 particle/accuracy_6_of_6‚1 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9936118125915527 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 jet/accuracy_7_of_7‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.45860469341278076 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.012717309407889843 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.02421383559703827 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.375 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5888683199882507 particle/accuracy_4_of_5‚1 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3263888955116272 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 5 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.98872971534729 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.5004292726516724 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.6228070259094238 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 * Purity/4FRt*SRqqt0FBt/FRt_purity‚0.125 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0515873022377491 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.6008896827697754 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.006366504356265068 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 particle/accuracy_4_of_4‚1 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.26613327860832214 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.21181397140026093 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.40351760387420654 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.17499502003192902|øV[o÷'¢¦ 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.10000000149011612 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9968725442886353 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.5313672423362732 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9907218813896179 ;"Purity/2FRt*SRqqt2FBt/event_purity‚0.013565892353653908 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6350575089454651 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5669006109237671 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4270833134651184 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 8 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.41535383462905884 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.3645833432674408 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_3‚0.24576181173324585 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.28074711561203003 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.19304554164409637 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9836171269416809 + Purity/1FRt0SRqqt3FBt/FRt_purity‚0.6875 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.6097221970558167 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4000000059604645 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_2‚0.7739431262016296 jet/accuracy_4_of_6‚0 'particle/f_score‚0.4074062705039978 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.18588514626026154 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 6 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.988304078578949 7 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.1896396428346634 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.1779792457818985 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3263888955116272 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9804672002792358 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9896175265312195 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.03425183519721031 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5917125344276428 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3214285671710968 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5638038516044617 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 particle/accuracy_6_of_6‚1 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9936118125915527 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 jet/accuracy_7_of_7‚0 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.45860469341278076 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.012717309407889843 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 :"Purity/0FRt2SRqqt1FBt/event_purity‚0.02421383559703827 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 ,"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.375 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5888683199882507 particle/accuracy_4_of_5‚1 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3263888955116272 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 5 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.98872971534729 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.5004292726516724 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.6228070259094238 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 * Purity/4FRt*SRqqt0FBt/FRt_purity‚0.125 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0515873022377491 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.6008896827697754 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.006366504356265068 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 particle/accuracy_4_of_4‚1 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.26613327860832214 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.21181397140026093 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.40351760387420654 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.17499502003192902 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.0963529720902443 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 7 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.1874316781759262 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.14902858436107635 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.0972222313284874 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.34702378511428833 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.4375 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.07926861196756363 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9659090638160706 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 'particle/accuracy‚0.850983738899231 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8124949932098389 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ,"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.625 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9886252284049988 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9925675988197327 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7424241900444031 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 jet/accuracy_6_of_6‚0 & Purity/1FRt0SRqqt4FBt/FRt_purity‚1 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.980235755443573 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9881795644760132 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 :"Purity/1FRt2SRqqt0FBt/event_purity‚0.00266400258988142 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.014328940771520138 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9937499761581421 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 +"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.25 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.24342657625675201 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 8"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.569968581199646 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13044582307338715 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.6315789222717285 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.6764705777168274 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5678910613059998 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.15804597735404968 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.42653509974479675 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.20296765863895416 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 9"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.4656223654747009 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 9"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.4690631926059723 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3565123677253723 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.40753257274627686 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 :"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.40740740299224854 8 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.33518514037132263 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 ;"Purity/1FRt2SRqqt1FBt/event_purity‚0.003448275849223137 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.43378376960754395 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.6034639477729797 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.15909090638160706 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 jet/accuracy_5_of_7‚0 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 jet/accuracy_5_of_6‚0 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9267138838768005 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9444443583488464 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.5846656560897827 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24497434496879578 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.0357520766556263 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.08452380448579788 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9744047522544861 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5705227255821228 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 & Purity/*FRt0SRqqt4FBt/FRt_purity‚1 ) Purity/4FRt1SRqqt0FBt/FRt_purity‚0.25 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9683907628059387 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9936092495918274 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 ) Purity/*FRt1SRqqt3FBt/FRt_purity‚0.25 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5602841973304749 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.3140290677547455 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.5921592712402344 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.012946957722306252 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9464285969734192 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 *loss/val_total_loss‚44.986293942523176 5 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.96648108959198 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5651731491088867 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.18831966817378998 particle/accuracy_1_of_6‚1 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5835162401199341 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.4735683798789978 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.2703857719898224 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 *jet/accuracy_1_of_4‚0.9284613132476807 particle/accuracy_2_of_5‚1 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.5024924874305725 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.14077557623386383 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.10905349999666214 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 particle/accuracy_6_of_7‚1 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 epoch‚8 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08057189732789993 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9864341616630554 *jet/accuracy_1_of_3‚0.8874834179878235 particle/accuracy_2_of_2‚1 +"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.25 particle/accuracy_4_of_6‚1 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 jet/accuracy_5_of_5‚0 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5678910613059998 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.14749863743782043 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.02017568051815033 +jet/accuracy_3_of_4‚0.05288912355899811 7 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.4000000059604645 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5381368398666382 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.35960084199905396 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.27616289258003235 :"Purity/0FRt1SRqqt2FBt/event_purity‚0.01867816224694252 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6445052623748779 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.2149708867073059 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5508672595024109 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 +particle/sensitivity‚0.3283292353153229 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 jet/accuracy_1_of_7‚1 jet/accuracy_3_of_7‚0 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 8"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.517579197883606 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_5‚0.9688428640365601 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.3318150043487549 *jet/accuracy_2_of_5‚0.5087172985076904 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.2916666567325592 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_2_of_3‚1 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.7708333134651184 % _timestamp‚1.7884535843322513e+09 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9896711111068726 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 particle/accuracy_5_of_6‚1 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.17557041347026825 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9533333778381348 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 particle/accuracy_3_of_6‚1 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.6234567165374756 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.47331154346466064 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ) Purity/4FRt1SRqqt*FBt/FRt_purity‚0.25 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.012373506091535091 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 particle/accuracy_1_of_1‚1 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.545882523059845 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.17499502003192902 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_6‚1 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.2418861985206604 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11062941700220108 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.38296133279800415 8 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.12260536849498749 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_3_of_4‚1 particle/accuracy_5_of_5‚1 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.37952542304992676 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 ,jet/accuracy_3_of_3‚0.019731007516384125 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.45643970370292664 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5082651972770691 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.2969917356967926 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9957951307296753 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.13026930391788483 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 6validation_average_jet_accuracy‚0.4523635804653168 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5783928036689758 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.13111525774002075 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 6 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.262037068605423 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 _runtime‚8699.806343938 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9913515448570251 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.5922619104385376 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 +jet/accuracy_2_of_2‚0.10539604723453522 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.6466667056083679 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 particle/accuracy_5_of_7‚1 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9872517585754395 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.44476747512817383 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 (jet/accuracy_1_of_1‚0.51680988073349 ,jet/accuracy_4_of_5‚0.004096262622624636 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.6226851940155029 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 trainer/global_step‚10880 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.0625 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5955635905265808 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9859274625778198 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 ,"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.375 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.4943965673446655 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.3927689790725708 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2046070098876953 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.03546099364757538 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.2490079551935196 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.998068630695343 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1776694804430008 8"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.566490888595581 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 _step‚894 8 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.12719298899173737 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.14139007031917572 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5701664090156555 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.29793471097946167 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5773777961730957 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.7708333134651184 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9914966225624084 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 ;"Purity/2FRt0SRqqt2FBt/event_purity‚0.015350878238677979 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 ) Purity/1FRt1SRqqt3FBt/FRt_purity‚0.25 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1480751931667328 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17811717092990875 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.08456198871135712 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.13805745542049408 8 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.13282422721385956 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.0972222313284874 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 jet/accuracy_4_of_7‚0 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.10115797072649002 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 9"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.4833333492279053 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_4‚1 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.9122807383537292 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9479167461395264 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6809585094451904 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.03912683576345444 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5545225739479065 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5541329383850098 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9628472924232483 ,jet/accuracy_4_of_4‚0.002807604381814599 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 9"Purity/0FRt1SRqqt*FBt/SRqqt_puritykŽ½‘Î#‚0.5699907541275024 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9809297919273376 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5672531127929688 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9867123961448669 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.0555555559694767 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt2FBt/event_purity‚0.01344086043536663 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.5325524210929871 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_5‚1 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5400616526603699 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5126854777336121 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 particle/accuracy_1_of_4‚1 6 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.420138955116272 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 8"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.606178343296051 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 *jet/accuracy_2_of_6‚0.3589743971824646 +jet/accuracy_3_of_6‚0.17307692766189575 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.44333332777023315 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.5901129841804504 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 * Purity/4FRt*SRqqt*FBt/FRt_purity‚0.125 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.034150224179029465 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5785714387893677 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 particle/accuracy_4_of_7‚1 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.45735853910446167 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.0941978245973587 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.025299709290266037 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.25540032982826233 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.4431818127632141 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.2380952537059784 jet/accuracy_6_of_7‚0 particle/accuracy_1_of_7‚1 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4851851761341095 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7121211886405945 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 9"Purity/0FRt0SRqqt3FBt/event_purity‚0.0416666679084301 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13611596822738647 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.18232321739196777 +jet/accuracy_3_of_5‚0.08856885880231857 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5325080156326294 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09128113836050034 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9765334129333496 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 ;"Purity/0FRt*SRqqt3FBt/event_purity‚0.035087719559669495 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5555999875068665 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9849824905395508 8 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.13705334067344666 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 8 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.18981482088565826 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5357142686843872 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 7 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.1533595621585846 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_7‚1 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 jet/accuracy_1_of_6‚1 jet/accuracy_2_of_7‚0.5 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.6388888955116272 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5887032747268677 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0030595569405704737 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 *jet/accuracy_2_of_4‚0.3569861948490143 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14148594439029694 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 *particle/specificity‚0.947686493396759 particle/accuracy_3_of_7‚1 particle/accuracy_7_of_7‚1 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.3566661775112152 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 particle/accuracy_1_of_3‚1 particle/accuracy_3_of_3‚1 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.10000000149011612 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6154837012290955 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.6034379601478577 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.13333334028720856 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.20955400168895721 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.25846946239471436 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.15214082598686218 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.25074276328086853 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5598620176315308 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5548207759857178 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1110139936208725 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.45016834139823914 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6277263760566711 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.34375þïÃa«’ø'‚(:‡ ÐÅæÔÅ—ý gpu.0.gpu‚7.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚34.0 gpu.0.powerWatts‚122.723' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.5446 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ï‚O!rù'‚(:ç ÐÅæÔœ¡ÿ# proc.memory.rssMB‚ 2571.62890625 disk.rbd1.out‚ 51.05859375 disk.dm-1.out‚ 337.1953125 disk.drbd2080.in‚0$ memory_percent‚93.74923706054688$ proc.memory.availableMB‚1024.125 proc.cpu.threads‚17 disk.sda2.in‚0 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.rbd0.in‚0* proc.memory.percent‚15.695977210998535 cpu‚25.3615295022331 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0$ disk./.usageGB‚303.8102607727051 disk.sda.out‚ 337.1953125 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0" disk.dm-0.out‚34930.2724609375 network.recv‚ 3.112853e+06 disk.sda2.out‚0 disk.drbd2266.out‚0* disk./.usagePercent‚20.388961514244013 disk.sda1.in‚0% disk.nvme0n1.out‚34930.2724609375 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 network.sent‚ 2.6748806e+07 disk.sda.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚34930.2724609375 disk.sda3.out‚ 337.1953125̽ïá“ú'‚(:ˆ ßÅæÔ쎄ý gpu.0.gpu‚46.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚182.978' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.5956 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¶·„›}û'‚(:ò ßÅæÔ¥ºÿ disk.dm-0.in‚0.078125& disk.nvme0n1.out‚35461.39892578125% disk./.usageGB‚303.83620834350586# disk.dm-0.out‚35461.39892578125 disk.drbd1716.out‚0 cpu‚25.360309877483843 disk.nvme0n1.in‚0.078125 network.recv‚ 3.11882e+06 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚35461.39892578125 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.out‚0 disk.rbd1.in‚0 proc.cpu.threads‚17 disk.drbd2266.out‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 337.7421875 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.drbd1716.in‚0 disk.sda.in‚ 0.01171875 disk.dm-1.out‚ 337.7421875$ memory_percent‚91.30966663360596 disk.rbd1.out‚ 51.1845703125 disk.sda1.out‚0 disk.dm-1.in‚ 0.01171875) proc.memory.availableMB‚ 1423.82421875* proc.memory.percent‚16.001558303833008 network.sent‚ 2.6893619e+07* disk./.usagePercent‚20.390702877491915 disk.drbd2265.out‚0 disk.sda.out‚ 337.7421875 disk.sda2.in‚0" proc.memory.rssMB‚ 2621.6953125‡g¾ù2ü'- _wandbruntime‚8717  _runtime‚8717BœJÇý'Á _step‚895 & lr-AdamW/pg1‚3.708254605383221e-06 & lr-AdamW/pg2‚3.708254605383221e-06 trainer/global_step‚10899 $ _timestamp‚1.788453602485937e+09 _runtime‚8717.957511878§”sáÌþ'Æ _runtime‚8717.957511878 _step‚895 & lr-AdamW/pg1‚3.708254605383221e-06 & lr-AdamW/pg2‚3.708254605383221e-06 trainer/global_step‚10899 $ _timestamp‚1.788453602485937e+09ÿó'³2ÿ'- _wandbruntime‚8717  _runtime‚8717ÁPè'€(âO N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 364960455 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚10899 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚635147644255744 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚693489236745728 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9142490846720 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 206051455 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚644289478739968 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚461 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚627590440522142 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚92724668 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 206031417 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 158908998 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 206031417 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚635147644255744 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 145065650 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚644289478739968 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9592440686592 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚92724670 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 158908998 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9108826837786 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚644290153105408 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 158909000 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚8717.958443365 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚703081483576832 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚703081677432320 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9108172400030 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9142490846720 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 % _timestamp‚1.7884536024866164e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 364940415 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 158909000 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚635147662258688 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚618482268122112 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 206051455 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚635147662258688 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚459 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚52340521 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚52340980 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 d•ò%(çO ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 206031417 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 158908998 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 206031417 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚635147644255744 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 145065650 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚644289478739968 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9592440686592 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚92724670 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 158908998 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9108826837786 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚644290153105408 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 158909000 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚8717.958443365 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 169915392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚703081483576832 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚703081677432320 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9108172400030 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9142490846720 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 % _timestamp‚1.7884536024866164e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 364940415 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 158909000 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚635147662258688 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚618482268122112 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 206051455 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚635147662258688 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚459 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚52340521 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚52340980 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 364960455 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚10899 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚635147644255744 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚693489236745728 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9142490846720 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 206051455 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UADeviceStatsMonitor.on_train_batch_start/allocated_bytÐÑ3€ûes.all.freed‚644289478739968 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚461 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚627590440522142 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚92724668 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0€…v}Ú2‚(- _wandbruntime‚8718  _runtime‚8718Öò‰ò&ƒ(ìM VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 364992951 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 364972885 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚618537172252672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9143310287872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9142653912064 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9143310287872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9593141453312 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚92732838 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚52345336 trainer/global_step‚10899 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚635204015297536 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚618537189292032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚644347325585408 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 145078174 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 145077725 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚693550863543296 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9108988834994 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚635203997294592 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚703144004996608 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9593298772480 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚635203997294592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚8718.663058125 _step‚897 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 158923016 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414^Êó÷&„(ñM TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 364992951 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 364972885 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚618537172252672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9143310287872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9142653912064 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9143310287872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9593141453312 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚92732838 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚52345336 trainer/global_step‚10899 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚635204015297536 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚618537189292032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚644347325585408 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 145078174 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 145077725 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚693550863543296 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 157319168 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9108988834994 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚635203997294592 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚703144004996608 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9593298772480 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚635203997294592 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚8718.663058125 _step‚897 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 158923016 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0~·À2…(- _wandbruntime‚8718  _runtime‚8718…X†(Ò 0loss/FRt2/assignment_loss‚3.1015725433826447 1loss/SRqqt3/assignment_loss‚1.022573502608449 1loss/FBt3/assignment_loss‚0.14721626926701017 /loss/FBt4/detection_loss‚3.1953505277633667 1loss/SRqqt4/detection_loss‚1.6164394555202986 /loss/FBt3/detection_loss‚0.7526805781510094 epoch‚9 /loss/FRt3/assignment_loss‚1.127728544473648 % _timestamp‚1.7884536031925251e+09 _step‚898 0loss/FRt1/assignment_loss‚3.4670260678976774 2loss/SRqqt4/assignment_loss‚1.4467158615589142 /loss/FRt4/detection_loss‚2.1059274667838834 .loss/FBt1/detection_loss‚3.504076225417001 1loss/SRqqt1/assignment_loss‚2.171876531626497 2loss/SRqqt2/assignment_loss‚1.4130273341484691 0loss/FBt2/assignment_loss‚0.5282514631175034 0loss/FBt4/assignment_loss‚0.3352435640990734 0loss/FBt1/assignment_loss‚0.3801092120920657 .loss/FRt1/detection_loss‚2.853980489075184 /loss/FRt3/detection_loss‚1.6091450238227845 /loss/FBt2/detection_loss‚1.7707112523225637 _runtime‚8718.664042512 0loss/FRt4/assignment_loss‚3.0694346135517336 .loss/FRt2/detection_loss‚2.173367844273647 0loss/SRqqt1/detection_loss‚2.655470473425729 0loss/SRqqt2/detection_loss‚1.605668973663579 0loss/SRqqt3/detection_loss‚1.015599753153888 %loss/total_loss‚43.06919357119662 trainer/global_step‚10899XGs†]‡(× 0loss/SRqqt1/detection_loss‚2.655470473425729 0loss/SRqqt2/detection_loss‚1.605668973663579 0loss/SRqqt3/detection_loss‚1.015599753153888 %loss/total_loss‚43.06919357119662 trainer/global_step‚10899 0loss/FRt2/assignment_loss‚3.1015725433826447 1loss/SRqqt3/assignment_loss‚1.022573502608449 1loss/FBt3/assignment_loss‚0.14721626926701017 /loss/FBt4/detection_loss‚3.1953505277633667 1loss/SRqqt4/detection_loss‚1.6164394555202986 /loss/FBt3/detection_loss‚0.7526805781510094 epoch‚9 /loss/FRt3/assignment_loss‚1.127728544473648 % _timestamp‚1.7884536031925251e+09 _step‚898 0loss/FRt1/assignment_loss‚3.4670260678976774 2loss/SRqqt4/assignment_loss‚1.4467158615589142 /loss/FRt4/detection_loss‚2.1059274667838834 .loss/FBt1/detection_loss‚3.504076225417001 1loss/SRqqt1/assignment_loss‚2.171876531626497 2loss/SRqqt2/assignment_loss‚1.4130273341484691 0loss/FBt2/assignment_loss‚0.5282514631175034 0loss/FBt4/assignment_loss‚0.3352435640990734 0loss/FBt1/assignment_loss‚0.3801092120920657 .loss/FRt1/detection_loss‚2.853980489075184 /loss/FRt3/detection_loss‚1.6091450238227845 /loss/FBt2/detection_loss‚1.7707112523225637 _runtime‚8718.664042512 0loss/FRt4/assignment_loss‚3.0694346135517336 .loss/FRt2/detection_loss‚2.173367844273647‚íq,Šžˆ(‚(:“ îÅæÔðŒý gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚173.269' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚34.653800000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¸žB}‰(‚(:ò îÅæÔ¯Ÿ€& disk.nvme0n1.out‚35541.68798828125 disk.dm-0.in‚0.078125* disk./.usagePercent‚20.391813694182982 disk.sda2.out‚0 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.drbd2265.in‚0 disk.drbd2265.out‚0# disk.dm-0.out‚35541.68798828125( proc.memory.availableMB‚ 1424.2890625* proc.memory.percent‚16.001558303833008 network.sent‚ 2.6899399e+07 disk.sda.out‚ 338.26953125 disk.rbd1.out‚ 51.1845703125 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0.078125 disk.sda1.out‚0 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 338.26953125$ memory_percent‚91.30682945251465 cpu‚25.3597444965499 network.recv‚ 3.123202e+06 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.sda2.in‚0 disk.drbd1716.out‚0( disk.nvme0c0n1.out‚35541.68798828125 disk.drbd2266.out‚0 disk.rbd0.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚303.8527603149414 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 338.26953125ê0’¹“Š(‚(:ˆ ýÅæÔÒ²‘ý gpu.0.gpu‚39.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.274' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.8548 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0„ì;pt‹(‚(:é ýÅæÔ¹÷üÿ disk.drbd2266.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 network.recv‚ 3.12841e+06 disk.drbd2080.out‚0 disk.sda3.out‚ 338.921875 disk.rbd1.out‚ 51.4189453125 disk.drbd1716.in‚0( proc.memory.availableMB‚ 1424.5390625 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda2.out‚0 disk.rbd1.in‚0 network.sent‚ 2.7007118e+07" disk./.usageGB‚303.86083984375 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0.078125& disk.nvme0n1.out‚36019.45751953125 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0* proc.memory.percent‚16.001558303833008* disk./.usagePercent‚20.392355918140506 disk.rbd0.in‚0# memory_percent‚91.3053035736084" proc.memory.rssMB‚ 2621.6953125( disk.nvme0c0n1.out‚36019.45751953125 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.dm-1.out‚ 338.921875 proc.cpu.threads‚17 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚36019.45751953125 disk.sda1.in‚0 disk.sda1.out‚0 cpu‚25.35924367999413 disk.sda.out‚ 338.921875 disk.drbd2265.out‚0ÜNÝP2Œ(- _wandbruntime‚8755  _runtime‚8755Ñ@ zÊ(Ä ' lr-AdamW/pg2‚3.4061969144995492e-06 trainer/global_step‚10949 % _timestamp‚1.7884536402661476e+09 _runtime‚8755.737741298 _step‚899 ' lr-AdamW/pg1‚3.4061969144995492e-06ÂBóAÏŽ(É _step‚899 ' lr-AdamW/pg1‚3.4061969144995492e-06 ' lr-AdamW/pg2‚3.4061969144995492e-06 trainer/global_step‚10949 % _timestamp‚1.7884536402661476e+09 _runtime‚8755.737741298ƒ¨9D2(- _wandbruntime‚8755  _runtime‚8755ó«Øõt(âO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9182820290048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚93133070 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚647148790853632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚450 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚621227491689472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚637965296198144 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚52565206 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 206977507 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 145697826 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 145698276 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚696571749688832 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚706207157359616 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚647148790853632 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 % _timestamp‚1.7884536402668517e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚52564758 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9183476652544 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 159609800 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 trainer/global_step‚10949 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 206957469 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 366567267 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 159609800 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9182820290048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 366587307 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚647148116488192 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚637965296198144 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚696571773628928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚630376482389550 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚637965314201088 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚621227474650112 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9183476652544 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048ÅÙV¬í'‘(çO ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9183476652544 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 159609800 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 trainer/global_step‚10949 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 206957469 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 366567267 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 159609800 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9182820290048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 366587307 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚647148116488192 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚637965296198144 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚696571773628928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚630376482389550 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚637965314201088 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚621227474650112 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9183476652544 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9182820290048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚93133070 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚647148790853632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚450 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚621227491689472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚637965296198144 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚52565206 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 206977507 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 145697826 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 145698276 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚696571749688832 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚706207157359616 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚647148790853632 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 % _timestamp‚1.7884536402668517e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚52564758 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0„' Ê2’(- _wandbruntime‚8756  _runtime‚8756"Ü~ò&“(ìM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚638021649236992 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚461 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚93141236 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚638021649236992 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9184296093696 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9184296093696 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 366599737 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚696633424366592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚638021667239936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9183639717888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 206975923 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7884536409697213e+09 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 366619803 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚459 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚52569230 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9636242243584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚630432874432398 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚621282395820032 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9150478612366 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚630432202955074 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 trainer/global_step‚10949 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 159623814 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚647205963333632 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 _step‚901 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9183639717888 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 206975923 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 145710927 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚706269474865152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 145710466 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚52569689 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚706269666610176 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 159623816 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 145710466 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚52569689 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚706269666610176 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 159623816 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9184296093696 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 366599737 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚696633424366592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚638021667239936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9183639717888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 206975923 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7884536409697213e+09 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 366619803 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚459 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 REDeviceStatsMonitor.on_train_batch_end/inactive_èyžÝŸsplit.small_pool.freed‚52569230 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9636242243584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚630432874432398 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚621282395820032 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9150478612366 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚630432202955074 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0…ÜÌ}Ð2•(- _wandbruntime‚8756  _runtime‚8756ÐûÚ²[–(Õ 1loss/SRqqt1/assignment_loss‚2.095277616381645 1loss/FBt3/assignment_loss‚0.13223642472698008 /loss/FBt4/detection_loss‚2.2768574158350625 2loss/SRqqt2/assignment_loss‚1.6071853651665151 2loss/SRqqt4/assignment_loss‚1.1516365256728762 0loss/FBt1/assignment_loss‚0.5516194617331962 .loss/FRt3/detection_loss‚1.684449685930178 0loss/SRqqt2/detection_loss‚1.693339346908033 1loss/SRqqt3/detection_loss‚1.2253747636812828 /loss/FBt2/detection_loss‚2.1218774084533965 %loss/total_loss‚42.55943064089108 .loss/FRt1/assignment_loss‚3.71951186215436 /loss/FRt4/assignment_loss‚2.663353452216024 0loss/FBt2/assignment_loss‚0.4367891111427785 epoch‚9 trainer/global_step‚10949 1loss/SRqqt1/detection_loss‚3.0133905827999117 2loss/SRqqt3/assignment_loss‚1.1344538070261478 0loss/SRqqt4/detection_loss‚1.418141554346529 /loss/FBt3/detection_loss‚0.7059583921943777 1loss/FBt4/assignment_loss‚0.14644655818119645 /loss/FRt1/detection_loss‚3.2186034189330206 0loss/FRt2/assignment_loss‚2.3515593065665317 0loss/FRt3/assignment_loss‚1.7328811431753224 /loss/FBt1/detection_loss‚3.2866956889629364 _runtime‚8756.443371441 /loss/FRt2/detection_loss‚2.1771920224508414 /loss/FRt4/detection_loss‚2.0145997262519337 _step‚902 % _timestamp‚1.7884536409718492e+09—u4y`—(Ú 2loss/SRqqt3/assignment_loss‚1.1344538070261478 0loss/SRqqt4/detection_loss‚1.418141554346529 /loss/FBt3/detection_loss‚0.7059583921943777 1loss/FBt4/assignment_loss‚0.14644655818119645 /loss/FRt1/detection_loss‚3.2186034189330206 0loss/FRt2/assignment_loss‚2.3515593065665317 0loss/FRt3/assignment_loss‚1.7328811431753224 /loss/FBt1/detection_loss‚3.2866956889629364 _runtime‚8756.443371441 /loss/FRt2/detection_loss‚2.1771920224508414 /loss/FRt4/detection_loss‚2.0145997262519337 _step‚902 % _timestamp‚1.7884536409718492e+09 1loss/SRqqt1/assignment_loss‚2.095277616381645 1loss/FBt3/assignment_loss‚0.13223642472698008 /loss/FBt4/detection_loss‚2.2768574158350625 2loss/SRqqt2/assignment_loss‚1.6071853651665151 2loss/SRqqt4/assignment_loss‚1.1516365256728762 0loss/FBt1/assignment_loss‚0.5516194617331962 .loss/FRt3/detection_loss‚1.684449685930178 0loss/SRqqt2/detection_loss‚1.693339346908033 1loss/SRqqt3/detection_loss‚1.2253747636812828 /loss/FBt2/detection_loss‚2.1218774084533965 %loss/total_loss‚42.55943064089108 .loss/FRt1/assignment_loss‚3.71951186215436 /loss/FRt4/assignment_loss‚2.663353452216024 0loss/FBt2/assignment_loss‚0.4367891111427785 epoch‚9 trainer/global_step‚10949 1loss/SRqqt1/detection_loss‚3.0133905827999117† eN¢“˜(‚(:ˆ ŒÆæÔæü×ý gpu.0.gpu‚40.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.441' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4882 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0½CR™(‚(:ô ŒÆæÔã´òÿ disk.sda.in‚ 0.01171875 disk.rbd1.out‚ 51.4189453125 disk.rbd0.in‚0 cpu‚25.358687918606417 proc.cpu.threads‚17 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2080.out‚0$ disk./.usageGB‚303.8434028625488 disk.sda.out‚ 339.37890625 disk.sda2.in‚0& disk.nvme0n1.out‚36061.43603515625$ memory_percent‚91.30721092224121 network.recv‚ 3.133504e+06# disk.dm-0.out‚36061.43603515625 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125( disk.nvme0c0n1.out‚36061.43603515625 disk.drbd2266.out‚0* proc.memory.percent‚16.001558303833008 disk.dm-0.in‚0.078125 disk.dm-1.out‚ 339.37890625 disk.drbd1716.out‚0 disk.drbd2266.in‚0( proc.memory.availableMB‚ 1424.2265625" proc.memory.rssMB‚ 2621.6953125 network.sent‚ 2.7013672e+07 disk.sda1.out‚0* disk./.usagePercent‚20.391185707701492 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 339.37890625 disk.sda1.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.rbd0.out‚0.078125É+úçžš(‚(:“ ›ÆæÔÊΣý gpu.0.gpu‚29.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚148.533' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚29.706599999999998 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚06êöÛ~›(‚(:ó ›ÆæÔ‹¼¸ÿ disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.dm-1.out‚ 340.17578125 disk.drbd2266.out‚0 disk.sda.out‚ 340.17578125& disk.nvme0n1.out‚36100.22509765625* proc.memory.percent‚16.001558303833008 disk.drbd2266.in‚0 disk.rbd0.in‚0# disk.dm-0.out‚36100.22509765625 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.sda1.out‚0 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-0.in‚0.078125 network.recv‚ 3.139124e+06 disk.sda3.out‚ 340.17578125 disk.rbd0.out‚0.078125 disk.sda.in‚ 0.01171875 disk.sda2.out‚0 disk.rbd1.out‚ 51.544921875 disk.drbd1716.out‚0$ memory_percent‚91.30721092224121* disk./.usagePercent‚20.391746620209013$ disk./.usageGB‚303.8517608642578( proc.memory.availableMB‚ 1424.2265625" proc.memory.rssMB‚ 2621.6953125 cpu‚25.358164738294803 proc.cpu.threads‚17 network.sent‚ 2.7121436e+07 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚36100.22509765625 disk.drbd1716.in‚0‘|Ö¿žœ(‚(:“ ªÆæÔâÝÖý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚186.548' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚37.309599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ÊN°|(‚(:ñ ªÆæÔù„¿ÿ* disk./.usagePercent‚20.391817022280932 disk.sda2.out‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 340.75390625 disk.sda1.out‚0 disk.drbd1716.out‚0' disk.nvme0c0n1.out‚36143.2958984375 disk.rbd0.out‚0.078125" proc.memory.rssMB‚ 2621.6953125* proc.memory.percent‚16.001558303833008 network.sent‚ 2.7128154e+07 network.recv‚ 3.14439e+06% disk./.usageGB‚303.85280990600586 disk.dm-1.in‚ 0.01171875" disk.dm-0.out‚36143.2958984375 disk.sda.in‚ 0.01171875) proc.memory.availableMB‚ 1424.22265625 disk.rbd1.out‚ 51.716796875 disk.dm-1.out‚ 340.75390625 disk.drbd1716.in‚0 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 cpu‚25.357668617882677 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚36143.2958984375 disk.sda1.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.sda3.out‚ 340.75390625$ memory_percent‚91.30723476409912 proc.cpu.threads‚17ç°]û2ž(- _wandbruntime‚8793  _runtime‚8793ûåÃüÇŸ(Á & lr-AdamW/pg2‚3.116629931528792e-06 trainer/global_step‚10999 % _timestamp‚1.7884536780537994e+09 _runtime‚ 8793.52537234 _step‚903 & lr-AdamW/pg1‚3.116629931528792e-06ÙάÌ (Æ trainer/global_step‚10999 % _timestamp‚1.7884536780537994e+09 _runtime‚ 8793.52537234 _step‚903 & lr-AdamW/pg1‚3.116629931528792e-06 & lr-AdamW/pg2‚3.116629931528792e-06‡Ä~a22¡(- _wandbruntime‚8793  _runtime‚8793Ä$¶è'¢(âO WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚93541468 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 368194119 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚633163195734074 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 207903559 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 207883521 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚709332740684288 _step‚904 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚709332553120256 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9190497516602 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚623972698217472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚640782948140544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9223806095872 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 160310598 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚699654286572032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 207883521 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9678430172160 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 160310598 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚640782948140544 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚633162524256958 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚623972681178112 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 160310600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9678266548224 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚699654310512128 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9224462458368 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚650006754236416 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9223806095872 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9189843078846 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 146331011 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9224462458368 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 368214159 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚640782966143488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 207903559 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 146331466 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚52789996 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚640782966143488 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 % _timestamp‚1.7884536780544875e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 8638156804¡M˜í'£(çO ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚709332553120256 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9190497516602 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚623972698217472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚640782948140544 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9223806095872 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 160310598 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚699654286572032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 207883521 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9678430172160 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 160310598 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚640782948140544 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚633162524256958 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚623972681178112 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 160310600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9678266548224 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚699654310512128 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9224462458368 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚650006754236416 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9223806095872 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9189843078846 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 146331011 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9224462458368 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 368214159 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚640782966143488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 207903559 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 146331466 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚52789996 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚640782966143488 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 % _timestamp‚1.7884536780544875e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚93541468 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 368194119 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚633163195734074 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 207903559 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚455 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 207883521 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚709332740684288 _step‚904ˆhÔê’2¤(- _wandbruntime‚8794  _runtime‚8794tÖ_ô°¥(ìM _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚699715937309696 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9679127913984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚ƒÉ|yB0 trainer/global_step‚10999 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚650063926703104 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9224625523712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 160324614 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 368226589 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚624027602348032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚640839319182336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9225281899520 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚633218244822482 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 368246655 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 368226589 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚640839301179392 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 % _timestamp‚1.7884536787678988e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 207901975 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚640839301179392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 368246655 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚52794555 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚93549638 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _runtime‚8794.239839783 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚640839319182336 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 160324616 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚699715961249792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 160324614 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚709395250677248 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 207901975 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚52794107 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚709395065223680 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚633218916299806 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9224625523712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9225281899520 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 207922039 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚93549636 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 146343743 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚650064601081856 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128–(”÷&¦(ñM B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 368226589 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚640839301179392 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 % _timestamp‚1.7884536787678988e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 207901975 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚640839301179392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 368246655 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚52794555 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚93549638 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 _runtime‚8794.239839783 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚640839319182336 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 160324616 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚699715961249792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 160324614 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚709395250677248 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 207901975 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚52794107 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚709395065223680 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚633218916299806 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9224625523712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9225281899520 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 207922039 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚93549636 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 146343743 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚650064601081856 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚699715937309696 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9679127913984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 trainer/global_step‚10999 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚650063926703104 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9224625523712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 160324614 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 368226589 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚624027602348032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚640839319182336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9225281899520 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚633218244822482 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 368246655 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872‰£sØ2§(- _wandbruntime‚8794  _runtime‚8794º'2_[¨(Õ trainer/global_step‚10999 /loss/FRt4/assignment_loss‚2.207038596920345 0loss/SRqqt1/detection_loss‚2.306344722256516 epoch‚9 _step‚906 %loss/total_loss‚41.91008794622674 2loss/SRqqt1/assignment_loss‚1.7947901406071403 2loss/SRqqt2/assignment_loss‚1.3391281536446427 /loss/FBt1/detection_loss‚2.4356780267424054 /loss/FBt3/assignment_loss‚0.236100280698193 /loss/FRt1/detection_loss‚2.9020675925108104 /loss/FRt2/detection_loss‚2.2252271622419357 1loss/SRqqt3/detection_loss‚1.0082491847561366 1loss/SRqqt4/detection_loss‚1.7413319010908406 0loss/FRt3/assignment_loss‚1.6605888470526664 /loss/FRt4/detection_loss‚2.3430274927421757 1loss/SRqqt2/detection_loss‚1.6662608691543903 0loss/FRt2/assignment_loss‚3.3644817769527435 0loss/FBt4/assignment_loss‚0.3202179243475503 .loss/FBt4/detection_loss‚2.288715889056524 0loss/FRt1/assignment_loss‚3.9641676866091213 2loss/SRqqt3/assignment_loss‚1.0934683503722078 2loss/SRqqt4/assignment_loss‚1.3037546651127438 0loss/FBt2/assignment_loss‚0.3351465092106082 /loss/FRt3/detection_loss‚1.8511927177708956 /loss/FBt2/detection_loss‚2.3728141099214555 /loss/FBt1/assignment_loss‚0.363528241713406 /loss/FBt3/detection_loss‚0.7867671047412866 % _timestamp‚1.7884536787696543e+09 _runtime‚8794.241165034`è1W`©(Ú .loss/FBt4/detection_loss‚2.288715889056524 0loss/FRt1/assignment_loss‚3.9641676866091213 2loss/SRqqt3/assignment_loss‚1.0934683503722078 2loss/SRqqt4/assignment_loss‚1.3037546651127438 0loss/FBt2/assignment_loss‚0.3351465092106082 /loss/FRt3/detection_loss‚1.8511927177708956 /loss/FBt2/detection_loss‚2.3728141099214555 /loss/FBt1/assignment_loss‚0.363528241713406 /loss/FBt3/detection_loss‚0.7867671047412866 % _timestamp‚1.7884536787696543e+09 _runtime‚8794.241165034 trainer/global_step‚10999 /loss/FRt4/assignment_loss‚2.207038596920345 0loss/SRqqt1/detection_loss‚2.306344722256516 epoch‚9 _step‚906 %loss/total_loss‚41.91008794622674 2loss/SRqqt1/assignment_loss‚1.7947901406071403 2loss/SRqqt2/assignment_loss‚1.3391281536446427 /loss/FBt1/detection_loss‚2.4356780267424054 /loss/FBt3/assignment_loss‚0.236100280698193 /loss/FRt1/detection_loss‚2.9020675925108104 /loss/FRt2/detection_loss‚2.2252271622419357 1loss/SRqqt3/detection_loss‚1.0082491847561366 1loss/SRqqt4/detection_loss‚1.7413319010908406 0loss/FRt3/assignment_loss‚1.6605888470526664 /loss/FRt4/detection_loss‚2.3430274927421757 1loss/SRqqt2/detection_loss‚1.6662608691543903 0loss/FRt2/assignment_loss‚3.3644817769527435 0loss/FBt4/assignment_loss‚0.3202179243475503ŠÞÐûSžª(‚(:“ ¹ÆæÔâ§ÿü gpu.0.gpu‚40.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚141.616' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.323200000000003 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Þ®oy«(‚(:î ¹ÆæÔÖ¯“ÿ disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 341.26171875 disk.drbd2265.in‚0 disk.rbd0.in‚0" disk.dm-0.out‚36182.7099609375) proc.memory.availableMB‚ 1424.19140625 cpu‚25.3571199558933 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.sda3.out‚ 341.26171875 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda2.in‚0% disk.nvme0n1.out‚36182.7099609375 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.sda.out‚ 341.26171875 disk.sda1.out‚0 network.recv‚ 3.147491e+06 disk.dm-1.in‚ 0.01171875 disk.sda2.out‚0 disk.rbd1.out‚ 51.779296875" proc.memory.rssMB‚ 2621.6953125* disk./.usagePercent‚20.393508208052122 disk.drbd1716.out‚0 disk.rbd1.in‚0 network.sent‚ 2.7133363e+07 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.drbd2080.in‚0' disk.nvme0c0n1.out‚36182.7099609375 disk.sda1.in‚0# memory_percent‚91.3074254989624* proc.memory.percent‚16.001558303833008$ disk./.usageGB‚303.8780097961426 disk.drbd2080.out‚0 proc.cpu.threads‚17^Ç“¬(‚(:ˆ ÈÆæÔ’‡²ý gpu.0.gpu‚49.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚172.641' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.5282 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÍgÌf{­(‚(:ð ÈÆæÔìÁœÿ disk.drbd1716.in‚0$ memory_percent‚91.30740165710449 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125 disk.dm-1.in‚ 0.01171875 disk.sda3.in‚ 0.01171875* proc.memory.percent‚16.001558303833008 disk.rbd0.in‚0 disk.sda1.out‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125" proc.memory.rssMB‚ 2621.6953125 network.recv‚ 3.154238e+06 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚36224.6025390625 cpu‚25.356598953391583 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚36224.6025390625 disk.dm-1.out‚ 341.69921875( proc.memory.availableMB‚ 1424.1953125% disk./.usageGB‚303.88744354248047 disk.dm-0.in‚0.078125 disk.drbd2080.out‚0 disk.sda3.out‚ 341.69921875* disk./.usagePercent‚20.394141314684298 disk.sda.in‚ 0.01171875 disk.sda1.in‚0 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 51.865234375" disk.dm-0.out‚36224.6025390625 disk.sda.out‚ 341.69921875 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 network.sent‚ 2.724232e+07}z3A2®(- _wandbruntime‚8831  _runtime‚8831.ý;(ʯ(Ä trainer/global_step‚11049 % _timestamp‚1.7884537163500743e+09 _runtime‚8831.821660221 _step‚907 ' lr-AdamW/pg1‚2.8396140017961823e-06 ' lr-AdamW/pg2‚2.8396140017961823e-06ÖnóøÏ°(É % _timestamp‚1.7884537163500743e+09 _runtime‚8831.821660221 _step‚907 ' lr-AdamW/pg1‚2.8396140017961823e-06 ' lr-AdamW/pg2‚2.8396140017961823e-06 trainer/global_step‚11049‹å b2±(- _wandbruntime‚8831  _runtime‚8831ž)–ÚV!²(âO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚53017050 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚702736847395328 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚11049 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚53016609 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9265448264192 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 161011400 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9721466471424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9721623803904 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚635948566124366 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚652865391984640 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 208829611 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9230678418254 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚93949868 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚712458289926656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚626717887706112 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 161011398 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 369841011 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚626717904745472 XIDeviceStatå/•¦’sMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 146966920 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9231332856010 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 161011400 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚643600618085888 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚643600618085888 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚93949870˜%iöí'³(çO MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚652866066350080 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚643600600082944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9264791901696 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚441 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚652865391984640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 369841011 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚626717904745472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 146966920 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9231332856010 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 161011400 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚643600618085888 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚643600618085888 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚93949870 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚53017050 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚702736847395328 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 trainer/global_step‚11049 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚53016609 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9265448264192 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 161011400 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9721466471424 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9721623803904 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚635948566124366 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚652865391984640 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 208829611 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9230678418254 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚93949868 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚712458289926656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚626717887706112 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 161011398 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚652923238830080 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚53021449 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 369873507 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚11049 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚643656971124736 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚702798474192896 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 161025414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 146979060 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚93958038 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚702798498132992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9722330321408 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 161025416 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 208828027 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 369853441 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚652922564451328 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 369853441 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9266267705344 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 161025416 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 146979487 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚712520987870720 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 _step‚909 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚652923238830080 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚53021449 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 369873507 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚11049 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚643656971124736 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚702798474192896 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 161025414 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 146979060 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚93958038 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚702798498132992 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9722330321408 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 161025416 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 208828027 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 369853441 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚652922564451328 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 369853441 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9266267705344 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 161025416 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 146979487 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚712520987870720 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 _step‚909 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚655724029732864 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚94358270 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚94358268 trainer/global_step‚11099 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 209755663 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 161712198 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚705819360338432 _runtime‚8869.474355281 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚646418270028288 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚655724704098304 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚705819384278528 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 209735625 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 147601724 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚629463094234112 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚638734607991774 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9272168195418 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9306434070016 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 161712200 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 371467863 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚53243456 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚912 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚655724029732864 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚715584018834432 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 161712198 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9764470931968 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9305777707520 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚646418270028288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9764634555904 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚646418252025344 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚638735279468890 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 371447823 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 161712200 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 209755663 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ’¡Eí'Å(çO M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 209735625 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 147601724 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚629463094234112 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚638734607991774 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9272168195418 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9306434070016 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 161712200 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 371467863 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚53243456 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚912 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚655724029732864 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚715584018834432 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 161712198 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9764470931968 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9305777707520 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚646418270028288 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9764634555904 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚646418252025344 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚638735279468890 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 371447823 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 161712200 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 209755663 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 147602177 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚646418252025344 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 209735625 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9306434070016 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚655724029732864 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚94358270 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚94358268 trainer/global_step‚11099 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 209755663 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 161712198 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚705819360338432 _runtime‚8869.474355281 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚646418270028288 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚655724704098304 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚705819384278528€Ca2Æ(- _wandbruntime‚8870  _runtime‚8870ðl(‘¥Ç(ëM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9272330192626 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚655781202199552 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 371480293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚646474623067136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚629518015404032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.788453754700576e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚646474623067136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 371480293 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9765314860032 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚655781876578304 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 161726216 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚94366438 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserveÈ:Ld_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 371500359 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚646474605064192 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚638791000034622 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚94366436 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9272984630590 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚53247946 _runtime‚8870.172600959 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 147614832 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 209774143 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 147614382 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 trainer/global_step‚11099 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚629517998364672 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚913 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚715646325936128 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚705881035016192 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚53248394 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 209774143 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964aœï7ö&È(ðM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9272330192626 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚655781202199552 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 371480293 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚646474623067136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚629518015404032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.788453754700576e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚646474623067136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 371480293 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9765314860032 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚655781876578304 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 161726216 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚94366438 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 371500359 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚646474605064192 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚638791000034622 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚94366436 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9272984630590 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚53247946 _runtime‚8870.172600959 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 147614832 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 209774143 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 147614382 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 trainer/global_step‚11099 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚629517998364672 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚913 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚715646325936128 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚705881035016192 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚53248394 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 209774143 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 174096384 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964‘e´ÎÚ2É(- _wandbruntime‚8870  _runtime‚8870Im·Y]Ê(× 0loss/FRt4/assignment_loss‚2.9181456143205815 2loss/SRqqt4/assignment_loss‚1.3879122959957881 1loss/FBt1/assignment_loss‚0.43982788454574884 .loss/FRt3/detection_loss‚2.150705465452468 /loss/FRt4/detection_loss‚2.1669866509058258 .loss/FBt2/detection_loss‚2.231554791331291 0loss/FRt1/assignment_loss‚4.0614066024621325 0loss/FBt2/assignment_loss‚0.3242360375297628 /loss/FBt4/detection_loss‚2.2895936146378517 trainer/global_step‚11099 /loss/FRt2/detection_loss‚2.2419618593576627 0loss/FRt2/assignment_loss‚2.6878762398010645 2loss/SRqqt1/assignment_loss‚2.2274770173761578 2loss/SRqqt2/assignment_loss‚1.3571854522106823 1loss/FBt4/assignment_loss‚0.36281303863506764 1loss/SRqqt3/detection_loss‚1.4074809617332873 _runtime‚8870.173729008 1loss/SRqqt4/detection_loss‚1.6261807551438159 1loss/FBt3/assignment_loss‚0.16906329836055906 0loss/FRt3/assignment_loss‚1.6383341436690473 2loss/SRqqt3/assignment_loss‚0.9554451241969101 .loss/FRt1/detection_loss‚3.320129118859768 0loss/SRqqt1/detection_loss‚2.633956966576753 epoch‚9 0loss/SRqqt2/detection_loss‚1.569924153111599 %loss/total_loss‚44.42490450792629 .loss/FBt1/detection_loss‚3.211559973657131 /loss/FBt3/detection_loss‚1.0451474480553349 % _timestamp‚1.7884537547021787e+09 _step‚914ÐÕ‰YbË(Ü .loss/FBt1/detection_loss‚3.211559973657131 /loss/FBt3/detection_loss‚1.0451474480553349 % _timestamp‚1.7884537547021787e+09 _step‚914 0loss/FRt4/assignment_loss‚2.9181456143205815 2loss/SRqqt4/assignment_loss‚1.3879122959957881 1loss/FBt1/assignment_loss‚0.43982788454574884 .loss/FRt3/detection_loss‚2.150705465452468 /loss/FRt4/detection_loss‚2.1669866509058258 .loss/FBt2/detection_loss‚2.231554791331291 0loss/FRt1/assignment_loss‚4.0614066024621325 0loss/FBt2/assignment_loss‚0.3242360375297628 /loss/FBt4/detection_loss‚2.2895936146378517 trainer/global_step‚11099 /loss/FRt2/detection_loss‚2.2419618593576627 0loss/FRt2/assignment_loss‚2.6878762398010645 2loss/SRqqt1/assignment_loss‚2.2274770173761578 2loss/SRqqt2/assignment_loss‚1.3571854522106823 1loss/FBt4/assignment_loss‚0.36281303863506764 1loss/SRqqt3/detection_loss‚1.4074809617332873 _runtime‚8870.173729008 1loss/SRqqt4/detection_loss‚1.6261807551438159 1loss/FBt3/assignment_loss‚0.16906329836055906 0loss/FRt3/assignment_loss‚1.6383341436690473 2loss/SRqqt3/assignment_loss‚0.9554451241969101 .loss/FRt1/detection_loss‚3.320129118859768 0loss/SRqqt1/detection_loss‚2.633956966576753 epoch‚9 0loss/SRqqt2/detection_loss‚1.569924153111599 %loss/total_loss‚44.42490450792629’µîö“Ì(‚(:ˆ „ÇæÔªƒæü gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.013' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0026 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0I÷Ô¡nÍ(‚(:ã „ÇæԪƎÿ disk.sda1.in‚0 disk.drbd2080.in‚0$ memory_percent‚91.30799770355225 disk.rbd0.out‚0.078125& disk.nvme0n1.out‚36379.95849609375 disk.dm-1.in‚ 0.01171875" proc.memory.rssMB‚ 2621.6953125* proc.memory.percent‚16.001558303833008 proc.cpu.threads‚17 disk.drbd2266.in‚0# disk.dm-0.out‚36379.95849609375 disk.sda1.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 52.169921875 disk.dm-1.out‚344.125 disk.drbd1716.in‚0 disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda2.out‚0 network.recv‚ 3.172649e+06 disk.sda3.in‚ 0.01171875 disk.sda.out‚344.125 disk.drbd2265.in‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚36379.95849609375 disk.sda2.in‚0 disk.nvme0c0n1.in‚0.078125) proc.memory.availableMB‚ 1424.09765625 network.sent‚ 2.73685e+07% disk./.usageGB‚303.93100357055664 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0 disk.drbd2265.out‚0 cpu‚25.354534772577445) disk./.usagePercent‚20.39706466471779 disk.sda3.out‚344.125 disk.sda.in‚ 0.01171875^-–ÿžÎ(‚(:“ “ÇæÔºáŽý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚125.156' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.031200000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÑjÒkÏ(‚(:à “ÇæÔÃÌùþ$ memory_percent‚91.30799770355225) proc.memory.availableMB‚ 1424.09765625 proc.cpu.threads‚17 network.recv‚ 3.177958e+06 disk.dm-0.in‚0.078125 disk.sda.out‚344.75 disk.sda3.in‚ 0.01171875% disk./.usageGB‚303.93568420410156 disk.sda2.in‚0 disk.drbd2080.out‚0 cpu‚25.35402387913672( disk./.usagePercent‚20.3973787859623 disk.sda1.out‚0 disk.rbd1.out‚ 52.326171875 disk.dm-1.out‚344.75 disk.drbd1716.out‚0* proc.memory.percent‚16.001558303833008 network.sent‚ 2.7476511e+07 disk.rbd0.out‚0.078125 disk.sda2.out‚0 disk.drbd2266.out‚0 disk.sda.in‚ 0.01171875& disk.nvme0n1.out‚36419.67724609375 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.drbd2266.in‚0# disk.dm-0.out‚36419.67724609375 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 disk.sda3.out‚344.75" proc.memory.rssMB‚ 2621.6953125 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0( disk.nvme0c0n1.out‚36419.67724609375 Â12Ð(- _wandbruntime‚8906  _runtime‚8906íQ÷ÆÑ(À % lr-AdamW/pg1‚2.32346359321216e-06 % lr-AdamW/pg2‚2.32346359321216e-06 trainer/global_step‚11149 % _timestamp‚1.7884537912886136e+09 _runtime‚8906.760217234 _step‚915ë¨ÖËÒ(Å _step‚915 % lr-AdamW/pg1‚2.32346359321216e-06 % lr-AdamW/pg2‚2.32346359321216e-06 trainer/global_step‚11149 % _timestamp‚1.7884537912886136e+09 _runtime‚8906.760217234“é{[{2Ó(- _wandbruntime‚8906  _runtime‚8906Á|>Ðç'Ô(áO WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚658582667481088 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 210681715 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚632208300762112 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9313003534826 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚658583341846528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 373094715 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚649235903967744 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9346763513344 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 _step‚916 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 210661677 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚708901921161728 $ _timestamp‚1.788453791289298e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚649235921970688 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚649235921970688 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 210681715 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚53471250 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 148238355 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9807755556352 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 210661677 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚641521321336298 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9807594029568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 373074675 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9346763513344 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚708901897221632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 162412998 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚11149 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9312349097070 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚658582667481088 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚718709491251200 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚435 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚632208317801472 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9347419875840 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 162412998 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 148237918 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚718709676718080 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 162413000 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0ý@ŸÑ~Õ(æO \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _runtime‚8906.761113265 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9347419875840 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚94766668 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9347419875840 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_tr\›¿în#ain_batch_start/active.large_pool.freed‚ 162412998 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 148237918 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚718709676718080 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 162413000 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚658582667481088 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 210681715 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚632208300762112 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9313003534826 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚658583341846528 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 373094715 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚649235903967744 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9346763513344 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 _step‚916 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 210661677 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚708901921161728 $ _timestamp‚1.788453791289298e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚649235921970688 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚649235921970688 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 210681715 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚53471250 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 148238355 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9807755556352 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 210661677 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚641521321336298 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9807594029568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 373074675 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9346763513344 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 185466880 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚708901897221632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 162412998 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 trainer/global_step‚11149 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9312349097070 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚658582667481088 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚718709491251200 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚435 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚632208317801472 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9313819969998 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9347582941184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚94774836 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 162427016 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚708963547959296 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 210700195 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 373127211 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚53475745 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚53476178 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 210700195 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚649292275009536 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _step‚917 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 210680131 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9808443242496 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _runtime‚8907.442186782 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚658640514326528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚658640514326528 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 162427014 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚718771991201792 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚708963571899392 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 162427014 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 210680131 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 162427016 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 148251016 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 373107145 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 373107145 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 148250581 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚649292275009536 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9348239316992 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚632263221932032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9347582941184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 373127211 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚649292257006592 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9348239316992 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚641576370424706 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430{wîK÷&Ø(ñM MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 210680131 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9808443242496 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 _runtime‚8907.442186782 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚658640514326528 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚658640514326528 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 162427014 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚718771991201792 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚708963571899392 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 162427014 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 210680131 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 162427016 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 148251016 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 373107145 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 373107145 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 148250581 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚649292275009536 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9348239316992 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚632263221932032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚435 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9347582941184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 373127211 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚649292257006592 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9348239316992 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚641576370424706 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9313165532034 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚94774838 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9313819969998 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9347582941184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚94774836 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 162427016 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚708963547959296 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 210700195 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 373127211 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚53475745 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚53476178 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚433 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 210700195 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚649292275009536 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _step‚917•@E 2Ù(- _wandbruntime‚8907  _runtime‚8907U©´eXÚ(Ò 1loss/SRqqt1/assignment_loss‚1.537920999933373 0loss/FBt1/assignment_loss‚0.2772540536452859 1loss/FBt3/assignment_loss‚0.17694893365544806 1loss/FBt4/assignment_loss‚0.07095514635251529 1loss/SRqqt4/detection_loss‚1.6000321613481412 _runtime‚8907.443442583 _step‚918 1loss/SRqqt4/assignment_loss‚1.627341514489112 epoch‚9 trainer/global_step‚11149 0loss/FBt2/assignment_loss‚0.6214997682251914 1loss/SRqqt3/detection_loss‚1.1897853889489827 /loss/FBt1/detection_loss‚2.9107778776775706 /loss/FBt2/detection_loss‚2.2142230918010077 .loss/FRt1/assignment_loss‚3.59174120426178 /loss/FRt3/detection_loss‚1.6349910759152129 /loss/FRt2/assignment_loss‚3.042510822797433 2loss/SRqqt3/assignment_loss‚0.9648591220276813 % _timestamp‚1.7884537919716094e+09 %loss/total_loss‚43.12557673524742 /loss/FRt1/detection_loss‚3.0115056987161988 /loss/FRt2/detection_loss‚2.4467496891052294 /loss/FRt3/assignment_loss‚1.820638731122017 0loss/SRqqt1/detection_loss‚2.529879725340641 1loss/SRqqt2/detection_loss‚2.1798649179664524 /loss/FRt4/assignment_loss‚2.590945999289668 2loss/SRqqt2/assignment_loss‚1.3262355419468472 .loss/FRt4/detection_loss‚2.461803344793098 /loss/FBt3/detection_loss‚0.6557227773126215 .loss/FBt4/detection_loss‚2.641389148575919ŸVW-]Û(× 2loss/SRqqt3/assignment_loss‚0.9648591220276813 % _timestamp‚1.7884537919716094e+09 %loss/total_loss‚43.12557673524742 /loss/FRt1/detection_loss‚3.0115056987161988 /loss/FRt2/detection_loss‚2.4467496891052294 /loss/FRt3/assignment_loss‚1.820638731122017 0loss/SRqqt1/detection_loss‚2.529879725340641 1loss/SRqqt2/detection_loss‚2.1798649179664524 /loss/FRt4/assignment_loss‚2.590945999289668 2loss/SRqqt2/assignment_loss‚1.3262355419468472 .loss/FRt4/detection_loss‚2.461803344793098 /loss/FBt3/detection_loss‚0.6557227773126215 .loss/FBt4/detection_loss‚2.641389148575919 1loss/SRqqt1/assignment_loss‚1.537920999933373 0loss/FBt1/assignment_loss‚0.2772540536452859 1loss/FBt3/assignment_loss‚0.17694893365544806 1loss/FBt4/assignment_loss‚0.07095514635251529 1loss/SRqqt4/detection_loss‚1.6000321613481412 _runtime‚8907.443442583 _step‚918 1loss/SRqqt4/assignment_loss‚1.627341514489112 epoch‚9 trainer/global_step‚11149 0loss/FBt2/assignment_loss‚0.6214997682251914 1loss/SRqqt3/detection_loss‚1.1897853889489827 /loss/FBt1/detection_loss‚2.9107778776775706 /loss/FBt2/detection_loss‚2.2142230918010077 .loss/FRt1/assignment_loss‚3.59174120426178 /loss/FRt3/detection_loss‚1.6349910759152129 /loss/FRt2/assignment_loss‚3.042510822797433–«$Y“Ü(‚(:ˆ ¢ÇæÔ¬ø›ý gpu.0.gpu‚37.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚163.321' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚32.6642 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0üX.¾Ý(‚(:î ¢ÇæÔÃÜÿ# disk.dm-0.out‚36462.95361328125 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0) disk./.usagePercent‚20.39792100991982 disk.sda.out‚ 345.6015625 disk.sda2.out‚0 disk.rbd1.out‚ 52.326171875 disk.dm-1.in‚ 0.01171875) proc.memory.availableMB‚ 1423.81640625 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.drbd1716.out‚0* proc.memory.percent‚16.0cwl»01558303833008% disk./.usageGB‚303.94376373291016 disk.sda3.out‚ 345.6015625 disk.drbd2080.in‚0 disk.rbd0.in‚0& disk.nvme0n1.out‚36462.95361328125 cpu‚25.3534840898721 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd1716.in‚0 network.recv‚ 3.183295e+06( disk.nvme0c0n1.out‚36462.95361328125 disk.drbd2266.in‚0 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.sda1.in‚0" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.dm-1.out‚ 345.6015625$ memory_percent‚91.30971431732178 network.sent‚ 2.748321e+07 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.0781256ž;“Þ(‚(:ˆ ±ÇæÔÆߢý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚186.233' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.2466 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0mÂcöyß(‚(:î ±ÇæÔØÛ¾ÿ disk.sda3.in‚ 0.01171875 cpu‚25.352974164445644 disk.dm-1.out‚ 346.171875 disk.nvme0n1.in‚0.078125& disk.nvme0n1.out‚36500.49658203125 disk.drbd2266.out‚0$ memory_percent‚91.30837917327881) proc.memory.availableMB‚ 1424.03515625 disk.dm-1.in‚ 0.01171875 network.recv‚ 3.188163e+06) disk./.usagePercent‚20.39762096908969% disk./.usageGB‚303.93929290771484 disk.rbd1.out‚ 52.388671875 disk.drbd2080.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125( disk.nvme0c0n1.out‚36500.49658203125 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.rbd0.in‚0" proc.memory.rssMB‚ 2621.6953125* proc.memory.percent‚16.001558303833008 disk.sda1.out‚0 disk.drbd1716.in‚0 disk.sda3.out‚ 346.171875 disk.drbd2266.in‚0# disk.dm-0.out‚36500.49658203125 network.sent‚ 2.7590899e+07 disk.drbd2265.out‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 346.171875 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.drbd2080.out‚0=–¸H“à(‚(:ˆ ÀÇæÔθ—ý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚136.249' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚27.2498 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0É6Ë~á(‚(:ó ÀÇæÔóó¡ÿ network.recv‚ 3.192911e+06 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.rbd1.out‚ 52.388671875 cpu‚25.352468805163326 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0( disk.nvme0c0n1.out‚36538.82080078125 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.drbd1716.in‚0$ memory_percent‚91.30988121032715 proc.cpu.threads‚17 disk.sda.out‚ 346.73046875 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.drbd1716.out‚0% disk./.usageGB‚303.96662521362305 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.nvme0n1.in‚0.078125( proc.memory.availableMB‚ 1423.7890625 network.sent‚ 2.7597339e+07 disk.sda3.out‚ 346.73046875 disk.dm-0.in‚0.078125# disk.dm-0.out‚36538.82080078125* proc.memory.percent‚16.001558303833008 disk.rbd0.out‚0.078125& disk.nvme0n1.out‚36538.82080078125 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 346.73046875) disk./.usagePercent‚20.39945526307252 disk.drbd2266.out‚0 disk.sda1.out‚0" proc.memory.rssMB‚ 2621.6953125ÓÇ«2â(- _wandbruntime‚8943  _runtime‚8943ñ°µÖÇã(Á _runtime‚8943.580523906 _step‚919 & lr-AdamW/pg1‚2.084436679332277e-06 & lr-AdamW/pg2‚2.084436679332277e-06 trainer/global_step‚11199 $ _timestamp‚1.788453828108905e+09 [éªÌä(Æ trainer/global_step‚11199 $ _timestamp‚1.788453828108905e+09 _runtime‚8943.580523906 _step‚919 & lr-AdamW/pg1‚2.084436679332277e-06 & lr-AdamW/pg2‚2.084436679332277e-06—è­±2å(- _wandbruntime‚8943  _runtime‚8943Û½&ç'æ(áO SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9850605673472 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 211587729 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚644307363203706 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9388405681664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 $ _timestamp‚1.788453828109657e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 374721567 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚721835039778304 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9850771394560 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 163113800 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9353184436478 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚439 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9388405681664 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚661441305229312 trainer/global_step‚11199 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚661441305229312 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚661441979594752 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9387749319168 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9387749319168 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚53699303 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 211607767 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚920 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 211607767 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚53698864 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚652053573913088 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 148873932 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚661441979594752 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚652053573913088 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 148874373 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚644306691726590 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚95175068 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚721835229439488 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚8943.581496663 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 163113798 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚95175070 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 163113800 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚634953524329472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944ÅRWì'ç(æO QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 211607767 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 _step‚920 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 211607767 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚53698864 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚652053573913088 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 148873932 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚661441979594752 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚652053573913088 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 148874373 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚644306691726590 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚95175068 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚721835229439488 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚8943.581496663 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 163113798 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚95175070 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 163113800 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚634953524329472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9850605673472 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 211587729 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚644307363203706 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9388405681664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 $ _timestamp‚1.788453828109657e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚441 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 374721567 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚721835039778304 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9850771394560 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 163113800 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9353184436478 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚439 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9388405681664 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚661441305229312 trainer/global_step‚11199 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚661441305229312 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚661441979594752 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9387749319168 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9387749319168 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚53699303 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844˜ZÕÁ°2è(- _wandbruntime‚8944  _runtime‚8944§¾ÑÑé(ìM OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 211626247 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 374733997 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 148887090 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚712046108782592 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚721897558386176 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚635008428460032 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 211606183 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚53703406 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _step‚921 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 374754063 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚661499152074752 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9354000871442 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 374754063 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 374733997 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚635008411420672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚661498477696000 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9354655309406 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884538288072898e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚652109908948992 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9389225122816 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9389225122816 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚644363083769438 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 211626247 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚448 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚644362412292114 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 163127816 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 211606183 KDDeviceStatsMonitor.on_t5^!rain_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚53703852 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9851635057152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚11199 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚95183236 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚652109926951936 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 148886642 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 374754063 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 374733997 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚635008411420672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚661498477696000 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚446 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9354655309406 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 % _timestamp‚1.7884538288072898e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚652109908948992 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9389225122816 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9389225122816 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚644363083769438 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 211626247 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚448 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚644362412292114 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 163127816 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 211606183 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚53703852 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9851635057152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚11199 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚95183236 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚652109926951936 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 148886642 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 211606183 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚53703406 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 _step‚921 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 374754063 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚661499152074752 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9354000871442 Pm ç'ö(áO ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚95583470 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚637698713818112 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 trainer/global_step‚11249 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9429391487488 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚664300617342976 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚715066994928128 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚664300617342976 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 149510375 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 163814598 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚664299942977536 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9394019775886 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚654871225855488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚647093405071114 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚637698730857472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚437 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚724960726967296 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚53926905 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 $ _timestamp‚1.788453865880735e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 212513781 _step‚924 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9428735124992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚654871207852544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚95583468 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚724960908239872 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚715066970988032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚664299942977536 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9394019775886 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚654871225855488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚647093405071114 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚637698730857472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚437 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚724960726967296 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚53926905 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 $ _timestamp‚1.788453865880735e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 212513781 _step‚924 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9428735124992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚654871207852544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 LBDeviceS”$ÍØBtatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚95583468 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚724960908239872 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚715066970988032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9429391487488 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚664300617342976 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚715066994928128 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚664300617342976 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 149510375 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 163814598 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚637753634988032 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9429554552832 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9395490648814 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 _step‚925 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 _runtime‚8982.060551997 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 376380915 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 376360849 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 149522661 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚647149125636846 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 212552299 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚654927560891392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚715128645665792 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚654927560891392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 212552299 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚95591636 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚53931025 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚725023222147072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 171999232 trainer/global_step‚11249 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 % _timestamp‚1.7884538665883932e+09 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 163828616 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 163828614 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚725023418086400 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚654927578894336 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9430210928640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9894772420608 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 212532235 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 376380915 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 163828616 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 149523111 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328t¹$C÷&ú(ñM _runtime‚8982.060551997 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 376380915 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 376360849 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 149522661 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚647149125636846 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 212552299 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚654927560891392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚715128645665792 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚654927560891392 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 212552299 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚95591636 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚53931025 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚725023222147072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 171999232 trainer/global_step‚11249 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 % _timestamp‚1.7884538665883932e+09 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚450 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 163828616 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 163828614 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚725023418086400 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚654927578894336 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9430210928640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9894772420608 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 212532235 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 376380915 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 163828616 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 149523111 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 195939328 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 163828614 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 212532235 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚637753634988032 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9429554552832 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9395490648814 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 _step‚925 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414r”2û(- _wandbruntime‚8982  _runtime‚8982V©X%Tü(Î /loss/FRt1/detection_loss‚3.2366697163808915 .loss/FRt3/assignment_loss‚2.18210521157907 1loss/SRqqt1/assignment_loss‚2.479001437779516 0loss/FBt3/assignment_loss‚0.2151882640348934 epoch‚9 trainer/global_step‚11249 $ _timestamp‚1.788453866590143e+09 2loss/SRqqt4/assignment_loss‚1.5174586828955907 /loss/FBt3/detection_loss‚0.5120222661430123 0loss/SRqqt1/detection_loss‚2.419132827781141 1loss/SRqqt2/detection_loss‚2.0496752585470674 _runtime‚8982.061659557 .loss/FRt2/detection_loss‚2.447078565271889 2loss/SRqqt2/assignment_loss‚1.7650977058708668 0loss/SRqqt3/detection_loss‚0.994340285414555 %loss/total_loss‚48.89826849104481 2loss/FBt4/assignment_loss‚0.030544988112524152 .loss/FBt2/detection_loss‚2.193205484322139 .loss/FBt4/detection_loss‚4.702612201372783 /loss/FRt4/assignment_loss‚2.639743673530492 1loss/SRqqt3/assignment_loss‚1.326815056858859 1loss/FBt1/assignment_loss‚0.49844786990433576 /loss/FRt4/detection_loss‚2.4426306680860845 1loss/SRqqt4/detection_loss‚1.5834231438525652 .loss/FBt1/detection_loss‚2.962418146431446 /loss/FRt1/assignment_loss‚4.279165529069447 /loss/FRt2/assignment_loss‚4.183211482152706 0loss/FBt2/assignment_loss‚0.5626256552391818 .loss/FRt3/detection_loss‚1.675654370413751 _step‚926ÁÀ¥Yý(Ó 2loss/SRqqt4/assignment_loss‚1.5174586828955907 /loss/FBt3/detection_loss‚0.5120222661430123 0loss/SRqqt1/detection_loss‚2.419132827781141 1loss/SRqqt2/detection_loss‚2.0496752585470674 _runtime‚8982.061659557 .loss/FRt2/detection_loss‚2.447078565271889 2loss/SRqqt2/assignment_loss‚1.7650977058708668 0loss/SRqqt3/detection_loss‚0.994340285414555 %loss/total_loss‚48.89826849104481 2loss/FBt4/assignment_loss‚0.030544988112524152 .loss/FBt2/detection_loss‚2.193205484322139 .loss/FBt4/detection_loss‚4.702612201372783 /loss/FRt4/assignment_loss‚2.639743673530492 1loss/SRqqt3/assignment_loss‚1.326815056858859 1loss/FBt1/assignment_loss‚0.49844786990433576 /loss/FRt4/detection_loss‚2.4426306680860845 1loss/SRqqt4/detection_loss‚1.5834231438525652 .loss/FBt1/detection_loss‚2.962418146431446 /loss/FRt1/assignment_loss‚4.279165529069447 /loss/FRt2/assignment_loss‚4.183211482152706 0loss/FBt2/assignment_loss‚0.5626256552391818 .loss/FRt3/detection_loss‚1.675654370413751 _step‚926 /loss/FRt1/detection_loss‚3.2366697163808915 .loss/FRt3/assignment_loss‚2.18210521157907 1loss/SRqqt1/assignment_loss‚2.479001437779516 0loss/FBt3/assignment_loss‚0.2151882640348934 epoch‚9 trainer/global_step‚11249 $ _timestamp‚1.788453866590143e+09žµ¬ì,’þ(‚(:‡ íÇæÔ˲¡ý gpu.0.gpu‚45.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚179.515' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.903 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Gé°¿|ÿ(‚(:ñ íÇæÔÑ»ÿ disk.drbd2265.in‚0 disk.drbd2080.out‚0$ memory_percent‚94.20344829559326 disk.nvme0n1.in‚0.078125 disk.drbd2266.out‚0( proc.memory.availableMB‚ 949.70703125 cpu‚25.350930000571065 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.dm-0.in‚0.078125 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.drbd1716.out‚0( disk.nvme0c0n1.out‚36664.18408203125 disk.drbd2266.in‚0 network.sent‚ 2.7717119e+07 disk.rbd1.out‚ 52.5771484375& disk.nvme0n1.out‚36664.18408203125 disk.nvme0c0n1.in‚0.078125 disk.rbd0.out‚0.078125# disk.dm-0.out‚36664.18408203125 disk.sda.out‚ 348.6484375 disk.sda1.in‚0 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.sda3.out‚ 348.6484375* proc.memory.percent‚16.001558303833008 network.recv‚ 3.206439e+06* disk./.usagePercent‚20.399055635311576 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 proc.cpu.threads‚17$ disk./.usageGB‚303.9606704711914 disk.sda1.out‚0 disk.dm-1.out‚ 348.6484375ÙšÍë“€)‚(:ˆ üÇæԬߖý gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚182.229' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.4458 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0}¹ïT|)‚(:ñ üÇæÔ–ç¤ÿ disk.rbd0.out‚0.078125# disk.dm-0.out‚36708.73876953125 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 network.recv‚ 3.21336e+06 disk.sda.in‚ 0.01171875* proc.memory.percent‚16.001558303833008 disk.sda1.in‚0 disk.sda3.out‚ 349.41796875 disk.drbd2266.in‚0$ disk./.usageGB‚303.9732475280762 disk.rbd1.in‚0 disk.dm-1.out‚ 349.41796875 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 proc.cpu.threads‚17 network.sent‚ 2.7826034e+07 disk.rbd0.in‚0" proc.memory.rssMB‚ 2621.6953125 disk.sda.out‚ 349.41796875 disk.rbd1.out‚ 52.6708984375& disk.nvme0n1.out‚36708.73876953125 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚36708.73876953125$ memory_percent‚94.20504570007324 cpu‚25.35043106918585* disk./.usagePercent‚20.399899692151966 disk.dm-0.in‚0.078125 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875' proc.memory.availableMB‚ 949.4453125†ôZï“‚)‚(:ˆ ‹ÈæÔäÇïü gpu.0.gpu‚45.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚187.078' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.4156 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0&C µ¹ƒ)‚(:å ‹ÈæÔ˜®• network.recv‚ 3.216851e+06 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125" proc.memory.rssMB‚ 2621.6953125* proc.memory.percent‚16.001558303“]À·833008 network.sent‚ 2.7831233e+07 disk.drbd2266.out‚0 disk.sda1.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 53.0146484375 disk.drbd1716.out‚0* disk./.usagePercent‚20.400351545449897 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.rbd1.in‚0% disk.nvme0n1.out‚36759.5380859375" disk./.usageGB‚303.97998046875 disk.sda3.out‚ 350.015625 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda.out‚ 350.015625 disk.drbd2265.in‚0$ memory_percent‚94.20509338378906 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2265.out‚0' disk.nvme0c0n1.out‚36759.5380859375 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0$ proc.memory.availableMB‚949.4375 cpu‚25.349949506282865 disk.dm-1.out‚ 350.015625" disk.dm-0.out‚36759.5380859375 disk.sda2.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125ÚÇ#2„)- _wandbruntime‚9019  _runtime‚9019Js¼Ê…)Ä ' lr-AdamW/pg1‚1.6447284862687778e-06 ' lr-AdamW/pg2‚1.6447284862687778e-06 trainer/global_step‚11299 % _timestamp‚1.7884539039447162e+09 _runtime‚9019.416341111 _step‚927—‹¤Wφ)É ' lr-AdamW/pg2‚1.6447284862687778e-06 trainer/global_step‚11299 % _timestamp‚1.7884539039447162e+09 _runtime‚9019.416341111 _step‚927 ' lr-AdamW/pg1‚1.6447284862687778e-06Ÿa­^i2‡)- _wandbruntime‚9019  _runtime‚90192r è'ˆ)âO JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚657688859794944 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚640443937385472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 164515398 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 164515398 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚54154437 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚649879446938522 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚667159255091200 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 trainer/global_step‚11299 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚640443920346112 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 164515400 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚657688859794944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚649878775461406 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 _runtime‚9019.417343373 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚928 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚667158580725760 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9435509553050 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚95991870 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚95991868 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9470377293312 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 213459871 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 377955231 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚728086154049536 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9469720930816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚667158580725760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚728086347905024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9469720930816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 377975271 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚54154891 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚657688877797888 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚718149507871232 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 213439833 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9434855115294 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 213459871 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚657688877797888 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 150146305 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 164515400 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 150146761 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9936816093696 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9936646178304:.9¹í'‰)çO UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚649878775461406 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚454 _runtime‚9019.417343373 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _step‚928 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚667158580725760 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚456 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9435509553050 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚95991870 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚95991868 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9470377293312 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 213459871 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 377955231 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚728086154049536 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9469720930816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚667158580725760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚728086347905024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 193855488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9469720930816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 377975271 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚54154891 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚657688877797888 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚718149507871232 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 213439833 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9434855115294 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 213459871 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚657688877797888 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 150146305 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 164515400 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 150146761 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9936816093696 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9936646178304 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚657688859794944 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚640443937385472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 164515398 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 164515398 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚54154437 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚649879446938522 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚667159255091200 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 trainer/global_step‚11299 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚640443920346112 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 164515400 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚657688859794944 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 “ÝAÆ2Š)- _wandbruntime‚9020  _runtime‚9020tßB‹ò&‹)ìM SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 378007767 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 377987701 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9435671550258 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚640498824476672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚728148681887744 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚640498841516032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚9020.148449424 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 213478351 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚657745230836736 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚718211158608896 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚667216427571200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚96000036 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9937523278848 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚667215753192448 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚96000038 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚718211182548992 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 378007767 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚728148861049856 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚667216427571200 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚657745212833792 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 213458287 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚54158870 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 150158906 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚54159313 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401ë@ï¯ Œ)ñM K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 213458287 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚657745230836736 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 155222016 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚667215753192448 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚96000038 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 179162112 cODeviceStatsMonitor.on_train_batch_,nq(ê#end/inactive_split_bytes.large_pool.allocated‚718211182548992 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 378007767 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚728148861049856 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚667216427571200 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚657745212833792 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 213458287 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚54158870 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 150158906 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚54159313 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 378007767 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 377987701 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9435671550258 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚640498824476672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚728148681887744 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚640498841516032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚9020.148449424 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 213478351 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚657745230836736 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚718211158608896 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚667216427571200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚96000036 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 9937523278848 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 379582083 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚670017218473984 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚455 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚652664817328814 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9475690454702 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 165216198 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 _step‚932 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚731211791859200 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 165216198 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚96400268 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9979906534400 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚54382714 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 150782984 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚643189143913472 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚643189126874112 % _timestamp‚1.7884539423704891e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚660506511737344 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 9979747104768 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 trainer/global_step‚11349 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚670017892839424 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 _runtime‚9057.842367783 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚670017218473984 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 165216200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 214365885 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚670017892839424 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 214365885 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚54382259 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚721232044754432 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚660506529740288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚652665488805930 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 165216200 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚731211975228928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 214385923 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9511363099136 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 150782527 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚670017218473984 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 165216200 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 214365885 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚670017892839424 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 214365885 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚54382259 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚721232044754432 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚660506529740288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚652665488805930 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 165216200 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚731211975228928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 214385923 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9511363099136 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 150782527 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 379582083 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚670017218473984 QIDeviceStatsMonitor.on_train_batch_startÓÒð‘3 /inactive_split.small_pool.current‚455 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚652664817328814 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9475690454702 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 165216198 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 _step‚932 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚731211791859200 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 165216198 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚96400268 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 9979906534400 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚54382714 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 150782984 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚643189143913472 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚643189126874112 % _timestamp‚1.7884539423704891e+09¤ñOXë2š)- _wandbruntime‚9058  _runtime‚9058ÒAñ&›)ëM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚54387282 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚11349 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚643244048044032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚643244031004672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚54386828 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚652720537894338 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9512182540288 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 379614553 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚660562864776192 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚96408436 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9980767547392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚456 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 150795264 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9512182540288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚660562882779136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 150795720 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 214404403 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚670075065319424 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 165230216 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚721293719432192 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 214384339 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9511526164480 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚454 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚670074390940672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 214404403 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚96408438 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚731274299428864 _runtime‚ 9058.55557022 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 165230214 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9477161327630 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚660562864776192 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 % _timestamp‚1.7884539430833447e+09 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 165230214 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚652721209371662 _step‚933 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 379634619ÂìV-ö&œ)ðM OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 165230214 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚652721209371662 _step‚933 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 379634619 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚54387282 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 trainer/global_step‚11349 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚643244048044032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚643244031004672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚54386828 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚652720537894338 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9512182540288 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 379614553 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚660562864776192 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚96408436 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 9980767547392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚456 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 150795264 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9512182540288 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚660562882779136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 150795720 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 214404403 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚670075065319424 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 165230216 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚721293719432192 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 214384339 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9511526164480 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚454 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚670074390940672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 214404403 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚96408438 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚731274299428864 _runtime‚ 9058.55557022 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 165230214 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9477161327630 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚660562864776192 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 % _timestamp‚1.7884539430833447e+09¥ Ž2ñ2)- _wandbruntime‚9058  _runtime‚9058VÅïVž)Ð 0loss/SRqqt1/detection_loss‚2.411724510292212 0loss/FRt3/assignment_loss‚1.9157187482889961 -loss/FBt1/assignment_loss‚0.6211232980402 1loss/FBt3/assignment_loss‚0.18709869329951853 /loss/FRt2/detection_loss‚2.3948810286819935 %loss/total_loss‚44.50581889540274 /loss/FRt1/assignment_loss‚3.652325799970916 1loss/SRqqt1/assignment_loss‚2.059760131355789 1loss/SRqqt3/assignment_loss‚1.251795098506685 2loss/SRqqt4/assignment_loss‚1.3776198314236743 1loss/FBt2/assignment_loss‚0.49135794490553053 .loss/FRt1/detection_loss‚2.849664084839098 .loss/FRt3/detection_loss‚1.693081673307746 0loss/SRqqt4/detection_loss‚1.621256171326552 0loss/FRt4/assignment_loss‚2.5066550434645958 2loss/SRqqt2/assignment_loss‚0.9863094248279345 .loss/FBt2/detection_loss‚3.122532472014427 0loss/FRt2/assignment_loss‚3.0824830057946118 /loss/FBt3/detection_loss‚0.6500681394923337 epoch‚9 _runtime‚ 9058.55684152 0loss/FBt4/assignment_loss‚0.6557121053338051 1loss/SRqqt2/detection_loss‚1.9131592396719783 1loss/SRqqt3/detection_loss‚1.4087332939921782 trainer/global_step‚11349 % _timestamp‚1.7884539430853298e+09 _step‚934 /loss/FRt4/detection_loss‚1.9814531604869892 /loss/FBt1/detection_loss‚2.8914242717954846 /loss/FBt4/detection_loss‚2.7798817242894853ÜIû[Ÿ)Õ 0loss/SRqqt1/detection_loss‚2.411724510292212 0loss/FRt3/assignment_loss‚1.9157187482889961 -loss/FBt1/assignment_loss‚0.6211232980402 1loss/FBt3/assignment_loss‚0.18709869329951853 /loss/FRt2/detection_loss‚2.3948810286819935 %loss/total_loss‚44.50581889540274 /loss/FRt1/assignment_loss‚3.652325799970916 1loss/SRqqt1/assignment_loss‚2.059760131355789 1loss/SRqqt3/assignment_loss‚1.251795098506685 2loss/SRqqt4/assignment_loss‚1.3776198314236743 1loss/FBt2/assignment_loss‚0.49135794490553053 .loss/FRt1/detection_loss‚2.849664084839098 .loss/FRt3/detection_loss‚1.693081673307746 0loss/SRqqt4/detection_loss‚1.621256171326552 0loss/FRt4/assignment_loss‚2.5066550434645958 2loss/SRqqt2/assignment_loss‚0.9863094248279345 .loss/FBt2/detection_loss‚3.122532472014427 0loss/FRt2/assignment_loss‚3.0824830057946118 /loss/FBt3/detection_loss‚0.6500681394923337 epoch‚9 _runtime‚ 9058.55684152 0loss/FBt4/assignment_loss‚0.6557121053338051 1loss/SRqqt2/detection_loss‚1.9131592396719783 1loss/SRqqt3/detection_loss‚1.4087332939921782 trainer/global_step‚11349 % _timestamp‚1.7884539430853298e+09 _step‚934 /loss/FRt4/detection_loss‚1.9814531604869892 /loss/FBt1/detection_loss‚2.8914242717954846 /loss/FBt4/detection_loss‚2.7798817242894853¦UÍpÒ“ )‚(:ˆ ¸ÈæÔïúšý gpu.0.gpu‚39.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚176.849' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.3698 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÜØŒN{¡)‚(:ð ¸ÈæÔ±òÿ disk.dm-1.out‚ 351.88671875 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.nvme0n1.in‚0.078125" proc.memory.rssMB‚ 2621.6953125 cpu‚25.348444340290854 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.sda2.in‚0 disk.rbd1.out‚ 53.6318359375* disk./.usagePercent‚20.401186130011524 disk.drbd1716.in‚0 disk.rbd0.out‚0.078125 disk.sda2.out‚0& proc.memory.availableMB‚ 949.109375 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875" disk.dm-0.out‚36884.1728515625 network.recv‚ 3.231864e+06% disk./.usageGB‚303.99241638183594 disk.sda1.in‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.sda3.out‚ 351.88671875 disk.dm-0.in‚0.078125 disk.sda.out‚ 351.88671875 proc.cpu.threads‚17 disk.nvme0c0n1.in‚0.078125 disk.sda1.out‚0% disk.nvme0n1.out‚36884.1728515625 network.sent‚ 2.7952458e+07' disk.nvme0c0n1.out‚36884.1728515625 disk.sda.in‚ 0.01171875$ memory_percent‚94.20709609985352* proc.memory.percent‚16.001558303833008vÓ➢)‚(:“ ÇÈæÔϤšý gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚161.457' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚32.291399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Õ.’7x£)‚(:í ÇÈæÔ‡±þþ network.sent‚ 2.8059245e+07* disk./.usagePercent‚20.401619294759456 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125 disk.sda2.in‚0 disk.rbd1.out‚ 53.8896484375$ disk.nvme0n1.out‚36926.255859375 disk.sda1.out‚0 network.recv‚ 3.236061e+06$ disk./.usageGB‚303.9988708496094 disk.dm-1.in‚ 0.01171875 disk.drbd2265.out‚0 disk.drbd2080.in‚0' proc.memory.availableMB‚ 949.3515625 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.sda.in‚ 0.01171875" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17 disk.dm-1.out‚ 352.63671875$ memory_percent‚94.20561790466309 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.sda.out‚ 352.63671875* proc.memory.percent‚16.001558303833008 cpu‚25.347930466718918 disk.drbd2265.in‚0 disk.sda3.out‚ 352.63671875& disk.nvme0c0n1.out‚36926.255859375 disk.rbd0.in‚0! disk.dm-0.out‚36926.255859375 disk.sda1.in‚0¯5: “¤)‚(:ˆ ÖÈæÔ´éÇþ gpu.0.gpu‚38.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚172.241' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.4482 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿¯äÂo¥)‚(:ä ÖÈæÔÆ’¡ÿ disk.sda2.in‚0$ disk.nvme0n1.out‚36951.220703125 disk.nvme0c0n1.in‚0.078125( proc.memory.availableMB‚ 949.59765625! disk.dm-0.out‚36951.220703125 disk.rbd1.out‚ 53.8896484375 disk.drbd2080.in‚0& disk.nvme0c0n1.out‚36951.220703125$ disk./.usageGB‚303.9820899963379 disk.sda.in‚ 0.01171875" proc.memory.rssMB‚ 2621.6953125 network.recv‚ 3.241301e+06 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.sda.out‚ 353.28125 disk.sda2.out‚0 disk.drbd2265.in‚0 network.sent‚ 2.8065885e+07 disk.drbd2266.in‚0 disk.sda1.in‚0$ memory_percent‚94.20411586761475 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875) disk./.usagePercent‚20.40049311761633 disk.drbd2266.out‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0* proc.memory.percent‚16.001558303833008 cpu‚25.347466171716317 proc.cpu.threads‚17 disk.dm-1.out‚ 353.28125 disk.sda3.out‚ 353.28125Ò 2¦)- _wandbruntime‚9096  _runtime‚9096]7Ûøȧ) _runtime‚9096.129034897 _step‚935 ' lr-AdamW/pg1‚1.2564487948328309e-06 ' lr-AdamW/pg2‚1.2564487948328309e-06 trainer/global_step‚11399 # _timestamp‚1.78845398065744e+09,WsºÍ¨)Ç _runtime‚9096.129034897 _step‚935 ' lr-AdamW/pg1‚1.2564487948328309e-06 ' lr-AdamW/pg2‚1.2564487948328309e-06 trainer/global_step‚11399 # _timestamp‚1.78845398065744e+09§â~Ž¶2©)- _wandbruntime‚9096  _runtime‚9096ÆK4ª)äO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 381208935 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚655450859196222 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚663324163679744 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚96808670 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10022829018112 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚655451530673338 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚663324163679744 E;DeviceStatsMonitor.on_train_batÁ NK¶"ch_start/allocation.all.peak‚24401 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10022980059136 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 174981120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚724314605577728 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 165916998 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚645934333402112 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9551692542464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 381228975 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚9096.130023953 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚445 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚54611733 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 215311975 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚11399 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚663324181682688 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9517180231866 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 165917000 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚936 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 165917000 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 215311975 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚672876530587648 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 165916998 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚645934350441472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 151420848 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884539806581454e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚672875856222208 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚447 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚96808668 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9516525794110÷èöï'«)éO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚672876530587648 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 165916998 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚645934350441472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 151420848 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 % _timestamp‚1.7884539806581454e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚672875856222208 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚447 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚96808668 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9516525794110 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 381208935 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚655450859196222 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚663324163679744 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚96808670 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10022829018112 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚655451530673338 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚663324163679744 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10022980059136 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 174981120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚724314605577728 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 165916998 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚645934333402112 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9551692542464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 381228975 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _runtime‚9096.130023953 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚445 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚54611733 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 215311975 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚11399 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚663324181682688 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9517180231866 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 165917000 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚936 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 165917000 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 215311975 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2¨®ìE{2¬)- _wandbruntime‚9097  _runtime‚9097Ç£aô&­)îM K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 165931014 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚724376256315392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 381241405 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚54616764 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚734400129558016 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚724376232375296 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚663380516718592 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚645989237532672 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884539816325543e+09 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9553168346112 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚9097.104533122 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚655506579761746 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚672933703067648 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 151433153 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9552511970304 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 165931016 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 215310391 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 165931014 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 215330455 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚672933703067648 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 381261471 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 215310391 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9552511970304 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9517342229074 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚96816836 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚663380534721536 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 381261471 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9517996667038 _step‚937 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 165931016 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚663380516718592 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚54616317 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 172870656 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚655507251239070 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9553168346112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 148930560 trainer/global_step‚11399 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚672933028688896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 151433602 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚734399956687360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844“ ‹®)óM C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚9097.104533122 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚655506579761746 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚672933703067648 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 151433153 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9552511970304 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 165931016 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 215310391 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 165931014 SEDeviceStatsMonitorJlq½ç.on_train_batch_end/allocation.small_pool.allocated‚ 215330455 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚672933703067648 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 381261471 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 215310391 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9552511970304 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9517342229074 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚96816836 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚663380534721536 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 381261471 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9517996667038 _step‚937 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 165931016 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚663380516718592 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚54616317 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 172870656 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚655507251239070 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9553168346112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 148930560 trainer/global_step‚11399 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚672933028688896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 151433602 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚734399956687360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 165931014 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚724376256315392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 381241405 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚54616764 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚734400129558016 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚724376232375296 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚663380516718592 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚645989237532672 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884539816325543e+09 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9553168346112©Ú›Ü12¯)- _wandbruntime‚9097  _runtime‚9097ÐÚ̈U°)Ï .loss/FRt4/detection_loss‚2.087262376262383 trainer/global_step‚11399 /loss/FRt2/assignment_loss‚2.945399461648403 /loss/FRt4/assignment_loss‚2.104386265711351 2loss/SRqqt1/assignment_loss‚1.9558481702750379 1loss/SRqqt4/detection_loss‚1.4062585799444107 .loss/FBt2/detection_loss‚2.488047733902931 0loss/FBt2/assignment_loss‚0.2669945280067577 0loss/SRqqt2/detection_loss‚1.840421892875849 .loss/FRt1/detection_loss‚3.194408175845941 /loss/FRt3/detection_loss‚1.5657527121188848 /loss/FBt1/detection_loss‚2.4600002434518604 0loss/FRt3/assignment_loss‚1.3340500943038776 0loss/FBt1/assignment_loss‚0.6055088986865561 0loss/SRqqt3/detection_loss‚1.464039877639152 %loss/total_loss‚43.07124337319194 % _timestamp‚1.7884539816341245e+09 _step‚938 /loss/FRt1/assignment_loss‚4.544196903705597 1loss/SRqqt2/assignment_loss‚1.168284940598316 0loss/SRqqt1/detection_loss‚2.908948865803805 _runtime‚9097.105618901 1loss/FBt4/assignment_loss‚0.14935188903473318 .loss/FBt4/detection_loss‚2.519696407020092 epoch‚9 1loss/SRqqt4/assignment_loss‚1.214783569227063 1loss/FBt3/assignment_loss‚0.15730422870198774 .loss/FRt2/detection_loss‚2.834595198241564 /loss/FBt3/detection_loss‚0.8584262489341199 2loss/SRqqt3/assignment_loss‚0.9972761112512671&fÏZ±)Ô _runtime‚9097.105618901 1loss/FBt4/assignment_loss‚0.14935188903473318 .loss/FBt4/detection_loss‚2.519696407020092 epoch‚9 1loss/SRqqt4/assignment_loss‚1.214783569227063 1loss/FBt3/assignment_loss‚0.15730422870198774 .loss/FRt2/detection_loss‚2.834595198241564 /loss/FBt3/detection_loss‚0.8584262489341199 2loss/SRqqt3/assignment_loss‚0.9972761112512671 .loss/FRt4/detection_loss‚2.087262376262383 trainer/global_step‚11399 /loss/FRt2/assignment_loss‚2.945399461648403 /loss/FRt4/assignment_loss‚2.104386265711351 2loss/SRqqt1/assignment_loss‚1.9558481702750379 1loss/SRqqt4/detection_loss‚1.4062585799444107 .loss/FBt2/detection_loss‚2.488047733902931 0loss/FBt2/assignment_loss‚0.2669945280067577 0loss/SRqqt2/detection_loss‚1.840421892875849 .loss/FRt1/detection_loss‚3.194408175845941 /loss/FRt3/detection_loss‚1.5657527121188848 /loss/FBt1/detection_loss‚2.4600002434518604 0loss/FRt3/assignment_loss‚1.3340500943038776 0loss/FBt1/assignment_loss‚0.6055088986865561 0loss/SRqqt3/detection_loss‚1.464039877639152 %loss/total_loss‚43.07124337319194 % _timestamp‚1.7884539816341245e+09 _step‚938 /loss/FRt1/assignment_loss‚4.544196903705597 1loss/SRqqt2/assignment_loss‚1.168284940598316 0loss/SRqqt1/detection_loss‚2.908948865803805ª瞙՞²)‚(:“ åÈæÔé‘ý gpu.0.gpu‚27.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚175.362' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚35.072399999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ä}g³)‚(:Ü åÈæÔö½¡ÿ disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.rbd1.in‚0 disk.nvme0n1.in‚0.078125' proc.memory.availableMB‚ 949.2890625* proc.memory.percent‚16.001558303833008 network.sent‚ 2.8071152e+07$ disk./.usageGB‚304.0191116333008 disk.drbd2266.out‚0 disk.dm-0.out‚ 36993.328125 cpu‚25.3469776678972 disk.drbd2080.in‚0# disk.nvme0c0n1.out‚ 36993.328125 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.dm-1.out‚ 354.171875 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2621.6953125 network.recv‚ 3.244775e+06* disk./.usagePercent‚20.402977670736135 disk.drbd2265.out‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 354.171875! disk.nvme0n1.out‚ 36993.328125$ memory_percent‚94.20599937438965 proc.cpu.threads‚17 disk.sda3.out‚ 354.171875 disk.sda1.out‚0 disk.rbd1.out‚ 54.0380859375 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.rbd0.out‚0.078125±“´)‚(:ˆ ôÈæÔâéõü gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.488' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2976 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚02#{uµ)‚(:ê ôÈæÔŽíöÿ disk.sda3.out‚ 354.73828125 disk.rbd0.out‚0.078125 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2265.in‚0( proc.memory.availableMB‚ 949.29296875 disk.sda1.out‚0 disk.rbd1.out‚ 54.1240234375$ memory_percent‚94.20597553253174 disk.drbd2266.out‚0 cpu‚25.346459721320116 network.sent‚ 2.8180123e+07 network.recv‚ 3.251574e+06 disk.drbd2080.out‚0% disk.nvme0c0n1.out‚37030.27734375 disk.sda3.in‚ 0.01171875 disk.dm-0.out‚37030.27734375 disk.rbd1.in‚0* proc.memory.percent‚16.001558303833008 disk.sda.in‚ 0.01171875 disk.sda.out‚ 354.73828125 disk.dm-1.out‚ 354.73828125" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17$ disk./.usageGB‚303.9967842102051# disk.nvme0n1.out‚37030.27734375 disk.drbd1716.out‚0) disk./.usagePercent‚20.40147925863823 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.dm-1.in‚ 0.01171875³wà“¶)‚(:ˆ ƒÉæÔºŒ²ý gpu.0.gpu‚50.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚181.082' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2164 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ÿ©Ih·)‚(:Ý ƒÉæÔ¶ÉÔÿ$ disk./.usageGB‚304.0108337402344 disk.dm-0.out‚ 37074.4609375 disk.sda.in‚ 0.01171875 disk.sda3.out‚ 355.40625 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.dm-0.in‚0.078125 disk.dm-1.out‚ 355.40625 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.rbd0.in‚0$ memory_percent‚94.20599937438965* proc.memory.percent‚16.001558303833008 disk.drbd2265.in‚0$ disk.nvme0c0n1.out‚ 37074.4609375 disk.drbd2266.out‚0' proc.memory.availableMB‚ 949.2890625 disk.sda2.in‚0 disk.rbd1.in‚0" disk.nvme0n1.out‚ 37074.4609375 disk.drbd2265.out‚0" proc.memory.rssMB‚ 2621.6953125 network.sent‚ 2.8185265e+07* disk./.usagePercent‚20.402422134386832 disk.sda.out‚ 355.40625 disk.drbd1716.in‚0 network.recv‚ 3.254734e+06 disk.sda1.in‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0.078125 disk.rbd1.out‚ 54.7646484375 disk.drbd2266.in‚0 cpu‚25.34597592370478 proc.cpu.threads‚17Šúº2¸)- _wandbruntime‚9135  _runtime‚9135Óýɹ)à ' lr-AdamW/pg1‚1.0816974602510365e-06 ' lr-AdamW/pg2‚1.0816974602510365e-06 trainer/global_step‚11449 $ _timestamp‚1.788454019540805e+09 _runtime‚9135.012366781 _step‚939œ·dËκ)È ' lr-AdamW/pg1‚1.0816974602510365e-06 ' lr-AdamW/pg2‚1.0816974602510365e-06 trainer/global_step‚11449 $ _timestamp‚1.788454019540805e+09 _runtime‚9135.012366781 _step‚939«þcð2»)- _wandbruntime‚9135  _runtime‚9135”8Äê'¼)äO N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚658237572540746 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 216217989 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚54841323 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚675735168335872 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚666141833625088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚727397142460928 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 382835787 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚97217070 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚648679539930112 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 166617798 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚727397118520832 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 216217989 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚675735168335872 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚666141815622144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚666141833625088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 trainer/global_step‚11449 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚675734493970432 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 152058393 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9557361133518 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 216238027 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 166617800 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚675734493970432 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 216238027 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9593334710784 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 152057929 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚737463236471296 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10065932483584 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9592678348288 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚462 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9558015571274 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚737463051004416 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚666141815622144 % _timestamp‚1.7884540195415745e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 _step‚940Ÿ8ˆ³Û½)éO F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 382835787 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚97217070 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚648679539930112 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 166617798 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚727397118520832 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 216217989 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚675735168335872 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚666141815622144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚666141833625088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 trainer/global_step‚11449 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚675734493970432 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 152058393 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9557361133518 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 216238027 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 166617800 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚675734493970432 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 216238027 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9593334710784 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 152057929 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚737463236471296 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10065932483584 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9592678348288 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚462 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 161526784 –=Í.^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9558015571274 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚737463051004416 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚666141815622144 % _timestamp‚1.7884540195415745e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 _step‚940 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚658237572540746 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 216217989 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚54841323 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚675735168335872 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚666141833625088 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚727397142460928¬b‘È/2¾)- _wandbruntime‚9135  _runtime‚9135„êg]ô&¿)îM XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9593497776128 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚737525748435456 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9558177568482 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 166631814 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 166631814 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚675791666437120 _runtime‚9135.741525627 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚97225236 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚648734444060672 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚658293293106478 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚465 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 382888323 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚666198168660992 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9593497776128 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚675792340815872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9594154151936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚463 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 % _timestamp‚1.7884540202692704e+09 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10066955236864 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9594154151936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚675792340815872 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 382868257 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 152070662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚97225238 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚658292621629154 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚666198186663936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 382888323 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚54845889 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚54845426 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚727458769258496 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚666198168660992 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚648734461100032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 216236443 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 216256507 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9558832006446 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10066787431936 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚11449 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚666198186663936 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808ÖK‹¾ù&À)óM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚675792340815872 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 382868257 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 152070662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚97225238 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 167804928 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 191745024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚658292621629154 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚666198186663936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 382888323 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚54845889 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚54845426 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚727458769258496 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚666198168660992 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚648734461100032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 216236443 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 216256507 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9558832006446 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10066787431936 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚11449 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚666198186663936 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9593497776128 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚737525748435456 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9558177568482 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 166631814 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 166631814 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚675791666437120 _runtime‚9135.741525627 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚97225236 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚648734444060672 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚658293293106478 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚465 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 382888323 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚666198168660992 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9593497776128 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚675792340815872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9594154151936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚463 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 % _timestamp‚1.7884540202692704e+09 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10066955236864 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9594154151936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872­v$˜é2Á)- _wandbruntime‚9135  _runtime‚9135ƒÚ,ZÂ)Ô 0loss/FRt4/assignment_loss‚2.8459328450262547 2loss/SRqqt3/assignment_loss‚1.0953500880859792 1loss/SRqqt4/detection_loss‚1.8010814609175378 $loss/total_loss‚46.1105624318196 _step‚942 .loss/FRt2/assignment_loss‚3.03280014583939 2loss/SRqqt1/assignment_loss‚2.4104127680000507 1loss/FBt3/assignment_loss‚0.20966050993238172 0loss/FBt4/assignment_loss‚0.4253460094332695 /loss/FRt4/detection_loss‚2.2847768800954023 /loss/FBt2/detection_loss‚2.6658314993748298 epoch‚9 /loss/FRt1/assignment_loss‚3.878779985286571 2loss/SRqqt2/assignment_loss‚1.1714960108713612 /loss/FBt3/detection_loss‚0.8386287491926006 /loss/FBt4/detection_loss‚3.4085783809423447 /loss/FRt3/detection_loss‚2.0464367706860815 1loss/SRqqt1/detection_loss‚3.1470695514427987 1loss/SRqqt3/detection_loss‚1.0923143874388188 1loss/FBt2/assignment_loss‚0.34261147785358215 /loss/FRt1/detection_loss‚3.0134435053224915 0loss/SRqqt2/detection_loss‚1.617186133304368 _runtime‚9135.742936962 0loss/FBt1/assignment_loss‚0.6314939446692165 % _timestamp‚1.7884540202711024e+09 0loss/FRt3/assignment_loss‚1.7939536387012118 1loss/SRqqt4/assignment_loss‚1.165100617503578 -loss/FRt2/detection_loss‚2.25289759588869 /loss/FBt1/detection_loss‚2.9393794760107994 trainer/global_step‚11449†‘î'_Ã)Ù 2loss/SRqqt2/assignment_loss‚1.1714960108713612 /loss/FBt3/detection_loss‚0.8386287491926006 /loss/FBt4/detection_loss‚3.4085783809423447 /loss/FRt3/detection_loss‚2.0464367706860815 1loss/SRqqt1/detection_loss‚3.1470695514427987 1loss/SRqqt3/detection_loss‚1.0923143874388188 1loss/FBt2/assignment_loss‚0.34261147785358215 /loss/FRt1/detection_loss‚3.0134435053224915 0loss/SRqqt2/detection_loss‚1.617186133304368 _runtime‚9135.742936962 0loss/FBt1/assignment_loss‚0.6314939446692165 % _timestamp‚1.7884540202711024e+09 0loss/FRt3/assignment_loss‚1.7939536387012118 1loss/SRqqt4/assignment_loss‚1.165100617503578 -loss/FRt2/detection_loss‚2.25289759588869 /loss/FBt1/detection_loss‚2.9393794760107994 trainer/global_step‚11449 0loss/FRt4/assignment_loss‚2.8459328450262547 2loss/SRqqt3/assignment_loss‚1.0953500880859792 1loss/SRqqt4/detection_loss‚1.8010814609175378 $loss/total_loss‚46.1105624318196 _step‚942 .loss/FRt2/assignment_loss‚3.03280014583939 2loss/SRqqt1/assignment_loss‚2.4104127680000507 1loss/FBt3/assignment_loss‚0.20966050993238172 0loss/FBt4/assignment_loss‚0.4253460094332695 /loss/FRt4/detection_loss‚2.2847768800954023 /loss/FBt2/detection_loss‚2.6658314993748298 epoch‚9 /loss/FRt1/assignment_loss‚3.878779985286571®3‹žÄ)‚(:“ ’ÉæÔ예ý gpu.0.gpu‚36.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚126.517' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.303399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0c;QtÅ)‚(:é ’ÉæÔå¼™ÿ network.recv‚ 3.260155e+06$ disk./.usageGB‚304.0496253967285 disk.sda2.out‚0# disk.nvme0n1.out‚37111.30859375 disk.drbd1716.in‚0 proc.cpu.threads‚17 disk.drbd1716.out‚0 disk.sda.out‚ 356.12109375 network.sent‚ 2.8192042e+07 disk.sda3.out‚ 356.12109375 disk.rbd0.out‚0.078125 disk.dm-0.out‚37111.30859375 disk.sda1.out‚0 disk.rbd1.in‚0 disk.sda2.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0$ memory_percent‚94.20619010925293 cpu‚25.345494899146566 disk.sda.in‚ 0.01171875 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 356.12109375 disk.drbd2265.in‚0) disk./.usagePercent‚20.40502547500252 disk.drbd2266.out‚0 disk.rbd1.out‚ 54.9365234375 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚0.078125% disk.nvme0c0n1.out‚37111.30859375' proc.memory.availableMB‚ 949.2578125" proc.memory.rssMB‚ 2621.6953125 disk.drbd2265.out‚0* proc.memory.percent‚16.001558303833008 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-0.in‚0.078125M$HèžÆ)‚(:“ ¡ÉæÔÀµåü gpu.0.gpu‚50.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚125.011' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.002199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÿh–~Ç)‚(:ó ¡ÉæÔÑ”·ÿ disk.drbd2080.out‚0 disk.dm-0.in‚0.078125# disk.dm-0.out‚37155.15380859375 disk.sda2.out‚0$ memory_percent‚94.20619010925293* proc.memory.percent‚16.001558303833008 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.sda.in‚ 0.01171875 network.recv‚ 3.265522e+06* disk./.usagePercent‚20.407959833359985$ disk./.usageGB‚304.0933494567871 disk.rbd1.in‚0& disk.nvme0n1.out‚37155.15380859375 disk.drbd2266.out‚0 disk.sda.out‚ 356.58203125 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.drbd2080.in‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.in‚0 disk.sda2.in‚0' proc.memory.availableMB‚ 949.2578125" proc.memory.rssMB‚ 2621.6953125 proc.cpu.threads‚17 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875( disk.nvme0c0n1.out‚37155.15380859375 disk.sda3.out‚ 356.58203125 cpu‚25.344984296389548 network.sent‚ 2.8299952e+07 disk.dm-1.out‚ 356.58203125 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.sda1.out‚0 disk.rbd1.out‚ 55.5537109375$ŸòÜ2È)- _wandbruntime‚9173  _runtime‚9173rd–ÈÉ) % _timestamp‚1.7884540581995137e+09 _runtime‚9173.671120401 _step‚943 & lr-AdamW/pg1‚9.199212557843736e-07 & lr-AdamW/pg2‚9.199212557843736e-07 trainer/global_step‚11499FAD·ÍÊ)Ç & lr-AdamW/pg1‚9.199212557843736e-07 & lr-AdamW/pg2‚9.199212557843736e-07 trainer/global_step‚11499 % _timestamp‚1.7884540581995137e+09 _runtime‚9173.671120401 _step‚943¯Pèk–2Ë)- _wandbruntime‚9173  _runtime‚9173:3*zKÌ)âO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 167318598 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚651424746458112 _runtime‚ 9173.67212627 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9633664154112 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚730479655404032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 217164079 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9634320516608 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚678593131718656 trainer/global_step‚11499 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 152694396 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚97625470 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚55069379 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚668959467564544 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 167318600 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚651424763497472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚740588899248128 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10109219904000 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9598196472926 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ]IDeviceStatsMonitor.on_tˆ,9rain_batch_start/active_bytes.large_pool.allocated‚668959485567488 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚740588711684096 $ _timestamp‚1.788454058200251e+09 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 167318600 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚453 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚661023614408154 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9598850910682 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚944 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 217164079 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚668959467564544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9634320516608 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 152694849 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚678593131718656 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9598850910682 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _step‚944 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 217164079 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚668959467564544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9634320516608 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 152694849 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚678593131718656 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 167318598 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚651424746458112 _runtime‚ 9173.67212627 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9633664154112 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚730479655404032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 217164079 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0°üBà62Î)- _wandbruntime‚9174  _runtime‚9174ÁïM”ô&Ï)îM B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 167332614 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 152707559 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9599667345854 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚740651416886272 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 trainer/global_step‚11499 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚651479667628032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 167332614 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 217182559 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 384515175 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚730541330081792 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55073468 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10109927388160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚97633636 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 % _timestamp‚1.7884540589086983e+09 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚669015838606336 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚669015838606336 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 384495109 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9634483581952 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 217182559 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚97633638 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚661079334973886 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 384515175 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 384495109 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚678650304185344 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚55073921 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚730541306141696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9599012907890 _step‚945 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 217162495 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 167332616 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 152707104 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚453 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10110086804480 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚651479650588672 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚9174.380797026 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 384515175 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚730541330081792 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55073468 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10109927388160 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚97633636 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 % _timestamp‚1.7884540589086983e+09 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚669015838606336 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚669015838606336 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 384495109 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9634483581952 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 217182559 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚97633638 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚661079334973886 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 384515175 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 384495109 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 TEDeviceStatsMonitor.on_train_batcÎlqÌ h_end/requested_bytes.large_pool.peak‚ 8838745088 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚678650304185344 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚55073921 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚730541306141696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9599012907890 _step‚945 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 217162495 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 167332616 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 152707104 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚453 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10110086804480 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚651479650588672 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚9174.380797026 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 167332614 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 152707559±d2Ò2Ñ)- _wandbruntime‚9174  _runtime‚9174Jðx[Ò)Õ /loss/FBt3/detection_loss‚0.8788018382708483 epoch‚9 % _timestamp‚1.7884540589108107e+09 0loss/FBt3/assignment_loss‚0.2684807713164306 1loss/SRqqt3/detection_loss‚1.1539781673446945 /loss/FBt1/detection_loss‚2.4740787115361957 1loss/SRqqt2/assignment_loss‚1.379065341384309 2loss/SRqqt3/assignment_loss‚0.9997330747706734 /loss/FRt1/detection_loss‚2.8979314883550007 1loss/SRqqt2/detection_loss‚2.0226346076261708 /loss/FRt1/assignment_loss‚4.293211201826732 0loss/FBt2/assignment_loss‚0.4656620738517377 -loss/FRt3/detection_loss‚2.02613806616711 1loss/SRqqt1/detection_loss‚2.6981037624420656 /loss/FBt4/detection_loss‚2.9309414525826774 0loss/FRt2/assignment_loss‚3.0793789447622095 0loss/FRt3/assignment_loss‚1.7522512360623008 2loss/SRqqt1/assignment_loss‚1.7980118264113703 1loss/FBt1/assignment_loss‚0.29682574937204564 1loss/FBt4/assignment_loss‚0.19974673027172685 .loss/FRt4/detection_loss‚2.183122882246971 1loss/SRqqt4/detection_loss‚1.5669812018934048 _runtime‚9174.382310142 /loss/FRt4/assignment_loss‚2.850153065919876 /loss/FBt2/detection_loss‚2.6556040807203813 .loss/FRt2/detection_loss‚2.407236284398018 &loss/total_loss‚44.580028214292625 1loss/SRqqt4/assignment_loss‚1.301955654759679 trainer/global_step‚11499 _step‚946¥Òá`Ó)Ú /loss/FBt3/detection_loss‚0.8788018382708483 epoch‚9 % _timestamp‚1.7884540589108107e+09 0loss/FBt3/assignment_loss‚0.2684807713164306 1loss/SRqqt3/detection_loss‚1.1539781673446945 /loss/FBt1/detection_loss‚2.4740787115361957 1loss/SRqqt2/assignment_loss‚1.379065341384309 2loss/SRqqt3/assignment_loss‚0.9997330747706734 /loss/FRt1/detection_loss‚2.8979314883550007 1loss/SRqqt2/detection_loss‚2.0226346076261708 /loss/FRt1/assignment_loss‚4.293211201826732 0loss/FBt2/assignment_loss‚0.4656620738517377 -loss/FRt3/detection_loss‚2.02613806616711 1loss/SRqqt1/detection_loss‚2.6981037624420656 /loss/FBt4/detection_loss‚2.9309414525826774 0loss/FRt2/assignment_loss‚3.0793789447622095 0loss/FRt3/assignment_loss‚1.7522512360623008 2loss/SRqqt1/assignment_loss‚1.7980118264113703 1loss/FBt1/assignment_loss‚0.29682574937204564 1loss/FBt4/assignment_loss‚0.19974673027172685 .loss/FRt4/detection_loss‚2.183122882246971 1loss/SRqqt4/detection_loss‚1.5669812018934048 _runtime‚9174.382310142 /loss/FRt4/assignment_loss‚2.850153065919876 /loss/FBt2/detection_loss‚2.6556040807203813 .loss/FRt2/detection_loss‚2.407236284398018 &loss/total_loss‚44.580028214292625 1loss/SRqqt4/assignment_loss‚1.301955654759679 trainer/global_step‚11499 _step‚946²ã ⶜Ô)‚(:‘ °ÉæÔùñýü gpu.0.gpu‚33.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚130.7' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚26.139999999999997 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿C6~Õ)‚(:ó °ÉæÔ…÷‡€ disk.sda.in‚ 0.01171875 disk.nvme0n1.in‚0.078125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2080.out‚0( disk.nvme0c0n1.out‚37194.63427734375" proc.memory.rssMB‚ 2621.6953125* disk./.usagePercent‚20.408374053550386$ disk./.usageGB‚304.0995216369629 disk.sda1.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 357.12890625 disk.sda2.out‚0& disk.nvme0n1.out‚37194.63427734375 disk.drbd2266.in‚0$ memory_percent‚94.20657157897949 disk.sda2.in‚0 disk.rbd0.in‚0 disk.rbd1.in‚0 disk.drbd2080.in‚0 disk.drbd2266.out‚0' proc.memory.availableMB‚ 949.1953125 cpu‚25.344527411203632 disk.sda1.out‚0 disk.dm-0.in‚0.078125 disk.dm-1.out‚ 357.12890625 network.sent‚ 2.8306433e+07 disk.sda.out‚ 357.12890625 disk.rbd1.out‚ 55.6396484375 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda3.in‚ 0.01171875* proc.memory.percent‚16.001558303833008 proc.cpu.threads‚17 network.recv‚ 3.270482e+06 disk.rbd0.out‚0.078125# disk.dm-0.out‚37194.63427734375¶‰•“Ö)‚(:ˆ ¿ÉæÔÏœ‚ý gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.654' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.9308 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0`\¦|×)‚(:ñ ¿ÉæÔ»ãÿ% disk./.usageGB‚304.14061737060547 disk.drbd2266.out‚0 disk.sda.out‚ 357.69140625 disk.sda1.in‚0 proc.cpu.threads‚17 disk.rbd0.out‚0.078125" disk.dm-0.out‚37242.0869140625 disk.sda2.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda1.out‚0 disk.sda2.out‚0* proc.memory.percent‚16.001582145690918 network.recv‚ 3.275758e+06 disk.sda3.out‚ 357.69140625' disk.nvme0c0n1.out‚37242.0869140625 disk.drbd1716.in‚0 network.sent‚ 2.8414283e+07 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125% disk.nvme0n1.out‚37242.0869140625 disk.drbd2080.in‚0# memory_percent‚94.2065954208374 cpu‚25.34402604516238 disk.sda3.in‚ 0.01171875 disk.dm-1.out‚ 357.69140625( proc.memory.availableMB‚ 949.19140625* disk./.usagePercent‚20.411132022716757 disk.rbd1.in‚0 disk.rbd1.out‚ 55.8271484375 disk.nvme0n1.in‚0.078125 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda.in‚ 0.01171875 disk.drbd2080.out‚0# proc.memory.rssMB‚ 2621.69921875¹Ï8ð“Ø)‚(:ˆ ÎÉæÔö‹Œý gpu.0.gpu‚39.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚184.261' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.8522 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0„²IzÙ)‚(:ï ÎÉæÔ–Îüþ disk.sda2.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0) disk./.usagePercent‚20.41420692921052 disk.sda.out‚ 358.35546875 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda3.in‚ 0.01171875( proc.memory.availableMB‚ 949.19140625 disk.rbd0.in‚0 disk.dm-0.in‚0.078125 disk.sda1.out‚0 disk.rbd1.out‚ 55.8271484375 cpu‚25.34357726334348 disk.nvme0n1.in‚0.078125' disk.nvme0c0n1.out‚37280.4150390625 proc.cpu.threads‚17 disk.rbd0.out‚0.078125 disk.sda3.out‚ 358.35546875* proc.memory.percent‚16.001582145690918" disk.dm-0.out‚37280.4150390625 disk.sda1.in‚0 disk.drbd2266.in‚0# proc.memory.rssMB‚ 2621.69921875 disk.drbd2080.in‚0$ disk./.usageGB‚304.1864356994629 disk.rbd1.in‚0% disk.nvme0n1.out‚37280.4150390625 disk.dm-1.out‚ 358.35546875 network.recv‚ 3.280861e+06 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2265.out‚0# memory_percent‚94.2065954208374 network.sent‚ 2.8421004e+07 disk.sda.in‚ 0.01171875ǽ>2Ú)- _wandbruntime‚9211  _runtime‚9211|)ÇÛ)Á & lr-AdamW/pg2‚7.711538953513772e-07 trainer/global_step‚11549 $ _timestamp‚1.788454096521361e+09 _runtime‚9211.993095072 _step‚947 & lr-AdamW/pg1‚7.711538953513772e-07~`xAÌÜ)Æ & lr-AdamW/pg1‚7.711538953513772e-07 & lr-AdamW/pg2‚7.711538953513772e-07 trainer/global_step‚11549 $ _timestamp‚1.788454096521361e+09 _runtime‚9211.993095072 _step‚947³<|$2Ý)- _wandbruntime‚9211  _runtime‚9211”¢£Äê'Þ)äO aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚733562192287232 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚671777119506944 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10152288880640 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚671777137509888 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚473 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10152114770944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 218070093 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚671777137509888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 153331482 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚98033870 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚681451769466880 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚681452443832320 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚9211.993966387 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 218090131 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 386089491 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚663809656275562 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚681451769466880 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 218070093 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚98033868 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚663808984798446 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9675306322432 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9674649959936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 168019400 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚55297614 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0Ŭüï'ß)éO \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚654169952986112 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 168019400 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚671777119506944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9675306322432 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚663808984798446 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9675306322432 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9674649959936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 168019400 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚55297614 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚733562192287232 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚671777119506944 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10152288880640 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚671777137509888 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚473 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10152114770944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 218070093 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚671777137509888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 153331482 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚98033870 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚681451769466880 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚681452443832320 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚9211.993966387 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 218090131 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 386089491 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚663809656275562 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚681451769466880 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 218070093 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚98033868 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 168033414 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 153344651 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ ô\Xêc$218108611 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9676125763584 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9676125763584 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 386142027 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚465 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 218108611 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9675469387776 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚654224874156032 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚681508941933568 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚733623866964992 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10153140894208 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚654224857116672 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10152977283584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚663864705363970 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚463 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚663865376841294 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚681509616312320 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 153344186 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 168033414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 218088547 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚671833490548736 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9640502685262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 168033416 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 218088547 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55302150 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 386142027 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9675469387776 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚743776820308480 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 168033416 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚98042036 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚671833490548736ø•ù&â)óM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚671833490548736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9639848247298 _runtime‚9212.723210771 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 168033414 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 153344651 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 218108611 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9676125763584 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9676125763584 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 386142027 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚465 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 218108611 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9675469387776 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚654224874156032 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚681508941933568 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚733623866964992 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10153140894208 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚654224857116672 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10152977283584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚663864705363970 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚463 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚663865376841294 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚681509616312320 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 153344186 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 168033414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 218088547 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚671833490548736 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9640502685262 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 168033416 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 218088547 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55302150 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 386142027 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 163610624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9675469387776 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚743776820308480 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 168033416 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚98042036 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0µ|»T2ã)- _wandbruntime‚9212  _runtime‚9212hdžZä)Ô 1loss/SRqqt4/detection_loss‚1.8689821765703314 /loss/FBt1/detection_loss‚2.7897883620527058 _runtime‚9212.724320385 0loss/FRt1/assignment_loss‚3.5311825688068685 0loss/FBt1/assignment_loss‚0.2910287214631353 1loss/SRqqt3/detection_loss‚1.0891707764187855 trainer/global_step‚11549 2loss/SRqqt3/assignment_loss‚0.8708035310473182 2loss/SRqqt4/assignment_loss‚1.1451288173888243 1loss/SRqqt2/detection_loss‚1.8917621657615755 /loss/FBt4/detection_loss‚3.7564884083611623 0loss/FBt4/assignment_loss‚0.4425044912168232 /loss/FRt1/detection_loss‚2.9869584830907674 epoch‚9 % _timestamp‚1.7884540972528296e+09 /loss/FRt4/assignment_loss‚2.853769724673413 1loss/SRqqt1/detection_loss‚2.5099129267036915 _step‚950 2loss/SRqqt2/assignment_loss‚1.4439131481254972 0loss/FBt2/assignment_loss‚0.9007623607221131 /loss/FBt2/detection_loss‚1.8987082472214332 /loss/FRt2/assignment_loss‚3.298361543062571 0loss/FBt3/assignment_loss‚0.2605370048376459 .loss/FRt2/detection_loss‚2.308403263422283 /loss/FBt3/detection_loss‚0.7773867153527239 2loss/SRqqt1/assignment_loss‚1.8309224657714367 .loss/FRt3/detection_loss‚1.598927687165829 -loss/FRt4/detection_loss‚2.42233081700954 %loss/total_loss‚44.45533469427079 0loss/FRt3/assignment_loss‚1.6876002880243155îHBÿ_å)Ù trainer/global_step‚11549 2loss/SRqqt3/assignment_loss‚0.8708035310473182 2loss/SRqqt4/assignment_loss‚1.1451288173888243 1loss/SRqqt2/detection_loss‚1.8917621657615755 /loss/FBt4/detection_loss‚3.7564884083611623 0loss/FBt4/assignment_loss‚0.4425044912168232 /loss/FRt1/detection_loss‚2.9869584830907674 epoch‚9 % _timestamp‚1.7884540972528296e+09 /loss/FRt4/assignment_loss‚2.853769724673413 1loss/SRqqt1/detection_loss‚2.5099129267036915 _step‚950 2loss/SRqqt2/assignment_loss‚1.4439131481254972 0loss/FBt2/assignment_loss‚0.9007623607221131 /loss/FBt2/detection_loss‚1.8987082472214332 /loss/FRt2/assignment_loss‚3.298361543062571 0loss/FBt3/assignment_loss‚0.2605370048376459 .loss/FRt2/detection_loss‚2.308403263422283 /loss/FBt3/detection_loss‚0.7773867153527239 2loss/SRqqt1/assignment_loss‚1.8309224657714367 .loss/FRt3/detection_loss‚1.598927687165829 -loss/FRt4/detection_loss‚2.42233081700954 %loss/total_loss‚44.45533469427079 0loss/FRt3/assignment_loss‚1.6876002880243155 1loss/SRqqt4/detection_loss‚1.8689821765703314 /loss/FBt1/detection_loss‚2.7897883620527058 _runtime‚9212.724320385 0loss/FRt1/assignment_loss‚3.5311825688068685 0loss/FBt1/assignment_loss‚0.2910287214631353 1loss/SRqqt3/detection_loss‚1.0891707764187855¶y2R™“æ)‚(:ˆ ÝÉæÔóØ¢ý gpu.0.gpu‚37.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚178.537' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚35.7074 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0$Ï©yç)‚(:î ÝÉæÔĤÄÿ disk.drbd1716.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚0.078125 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda.out‚ 358.93359375 disk.drbd1716.in‚0 disk.sda.in‚ 0.01171875 proc.cpu.threads‚17 network.sent‚ 2.8426299e+07* disk./.usagePercent‚20.422971091137697 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.out‚ 55.8896484375 disk.dm-0.in‚0.078125$ memory_percent‚93.87943744659424) proc.memory.availableMB‚ 1002.79296875$ disk./.usageGB‚304.3170280456543 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 358.93359375 disk.drbd2080.in‚0& disk.nvme0c0n1.out‚37327.802734375 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 358.93359375! disk.dm-0.out‚37327.802734375 disk.rbd0.out‚0.078125* proc.memory.percent‚15.674829483032227 network.recv‚ 3.28414e+06$ disk.nvme0n1.out‚37327.802734375 disk.drbd2266.in‚0" proc.memory.rssMB‚ 2568.1640625 disk.drbd2265.out‚0 disk.drbd2266.out‚0 disk.sda1.out‚0 cpu‚25.343098183420942˜Œžè)‚(:“ ìÉæÔâéŸý gpu.0.gpu‚28.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚128.652' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.730399999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ËïÈØué)‚(:ê ìÉæÔ«ûþþ network.recv‚ 3.291197e+06 disk.rbd1.in‚0$ disk.nvme0n1.out‚37363.767578125 disk.drbd2265.out‚0 disk.sda1.in‚0 disk.sda3.out‚ 359.5390625 disk.drbd2266.out‚0 disk.rbd0.in‚0! disk.dm-0.out‚37363.767578125( proc.memory.availableMB‚ 1002.7890625 disk.dm-1.in‚ 0.01171875 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 cpu‚25.342653969777448 network.sent‚ 2.8535453e+07 disk.drbd2265.in‚0 disk.sda.out‚ 359.5390625 proc.cpu.threads‚17 disk.drbd1716.in‚0) disk./.usagePercent‚20.42921562691289 disk.nvme0c0n1.in‚0.078125 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0.078125$ disk./.usageGB‚304.4100761413574 disk.sda2.out‚0 disk.rbd1.out‚ 55.8896484375 disk.sda.in‚ 0.01171875$ memory_percent‚93.87946128845215 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125 disk.dm-1.out‚ 359.5390625& disk.nvme0c0n1.out‚37363.767578125 disk.sda1.out‚0" proc.memory.rssMB‚ 2568.1640625* proc.memory.percent‚15.674829483032227 disk.drbd1716.out‚0nQ(2ê)- _wandbruntime‚9250  _runtime‚9250ËqKZÈë) & lr-AdamW/pg2‚6.354263818469983e-07 trainer/global_step‚11599 % _timestamp‚1.7884541352039192e+09 _runtime‚9250.675509186 _step‚951 & lr-AdamW/pg1‚6.354263818469983e-07Ñbp²Íì)Ç _step‚951 & lr-AdamW/pg1‚6.354263818469983e-07 & lr-AdamW/pg2‚6.354263818469983e-07 trainer/global_step‚11599 % _timestamp‚1.7884541352039192e+09 _runtime‚9250.675509186·•Ô;22í)- _wandbruntime‚9250  _runtime‚9250h!¼?î)ãO QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 168720200 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚454 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 168720198 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 218996145 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚684310407215104 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚11599 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚674594771449344 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚684311081580544 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 387716343 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚674594771449344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚666595698142970 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚452 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _step‚952 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9679867151742 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 168720200 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚98442270 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚55526505 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚9250.676496319 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 153968775 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚55526053 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.788454135204636e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9716292128256 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 219016183 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 218996145 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚00ÃH_î'ï)èO TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 165721088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _runtime‚9250.676496319 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 153968775 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚55526053 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.788454135204636e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9716292128256 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 219016183 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 218996145 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 168720200 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚454 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 168720198 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 218996145 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚684310407215104 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚11599 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚674594771449344 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 189661184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚684311081580544 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 387716343 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚674594771449344 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚666595698142970 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚452 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 _step‚952 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9679867151742 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 168720200 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚98442270 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚55526505 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1¸º÷¨Æ2ð)- _wandbruntime‚9251  _runtime‚9251ÝN‘=ô&ñ)îM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 168734214 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚656970080684032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 387748813 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚666651418708702 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚674651142491136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚674651142491136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 168734216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 153980987 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 219014599 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9681338024670 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 219034663 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 168734216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 trainer/global_step‚11599 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 219034663 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9716455193600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚98450438 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 168734214 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚656970063644672 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚9251.407227084 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 219014599 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10196245782016 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 387768879 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9717111569408 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚684368254060544 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 % _timestamp‚1.7884541359351468e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 387768879 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚684368254060544 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9717111569408 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55530551 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10196405198336 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚666650747231378 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚746902809046528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚953 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚736706403848192 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9680683586706 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚98450436 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚684367579681792 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚736706379908096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414Ú„{$ò)óM FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚736706403848192 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9680683586706 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚98450436 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚684367579681792 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚736706379908096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 168734214 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚656970080684032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 387748813 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚666651418708702 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚674651142491136 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚674651142491136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 168734216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 153980987 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 219014599 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9681338024670 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 219034663 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 168734216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 trainer/global_step‚11599 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 219034663 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9716455193600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚98450438 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 168734214 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚656970063644672 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 _runtime‚9251.407227084 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 219014599 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10196245782016 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 387768879 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9717111569408 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚684368254060544 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 % _timestamp‚1.7884541359351468e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 387768879 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚684368254060544 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9717111569408 REDeviceStatsMonitor.on_train_bsŸ‹áÚatch_end/inactive_split.small_pool.freed‚55530551 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10196405198336 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚666650747231378 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚746902809046528 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 _step‚953 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064¹΀1Œ2ó)- _wandbruntime‚9251  _runtime‚9251¦ê.ÐZô)Ô 1loss/SRqqt4/assignment_loss‚1.292714363662526 0loss/FBt3/assignment_loss‚0.1853695550758361 2loss/SRqqt2/assignment_loss‚1.5057149759597248 /loss/FBt2/detection_loss‚1.6626302931043837 _step‚954 0loss/FRt3/assignment_loss‚1.7339259757185883 0loss/FBt1/assignment_loss‚0.4685464867896397 .loss/FRt1/detection_loss‚3.247189428125109 1loss/SRqqt4/detection_loss‚1.7339471593343962 0loss/FRt4/assignment_loss‚2.6127154316220964 /loss/FBt1/detection_loss‚2.6444021738492527 trainer/global_step‚11599 2loss/SRqqt1/assignment_loss‚1.8819136712057838 /loss/FRt2/detection_loss‚1.9336468541040652 /loss/FBt3/detection_loss‚0.7616682152361854 %loss/total_loss‚43.23856963173067 $ _timestamp‚1.788454135937098e+09 /loss/FRt2/assignment_loss‚3.129420137986904 1loss/SRqqt3/assignment_loss‚1.071388105869234 1loss/FBt4/assignment_loss‚0.36393337137599246 epoch‚9 _runtime‚9251.408608973 /loss/FRt1/assignment_loss‚3.317117997578212 /loss/FRt4/detection_loss‚2.3514668112810777 1loss/SRqqt2/detection_loss‚1.8173722826772265 0loss/SRqqt3/detection_loss‚1.124687996175554 /loss/FBt4/detection_loss‚3.6247353553771973 1loss/FBt2/assignment_loss‚0.29835799450258527 /loss/FRt3/detection_loss‚1.9371051874784928 0loss/SRqqt1/detection_loss‚2.538599807640602i˜Û_õ)Ù 2loss/SRqqt2/assignment_loss‚1.5057149759597248 /loss/FBt2/detection_loss‚1.6626302931043837 _step‚954 0loss/FRt3/assignment_loss‚1.7339259757185883 0loss/FBt1/assignment_loss‚0.4685464867896397 .loss/FRt1/detection_loss‚3.247189428125109 1loss/SRqqt4/detection_loss‚1.7339471593343962 0loss/FRt4/assignment_loss‚2.6127154316220964 /loss/FBt1/detection_loss‚2.6444021738492527 trainer/global_step‚11599 2loss/SRqqt1/assignment_loss‚1.8819136712057838 /loss/FRt2/detection_loss‚1.9336468541040652 /loss/FBt3/detection_loss‚0.7616682152361854 %loss/total_loss‚43.23856963173067 $ _timestamp‚1.788454135937098e+09 /loss/FRt2/assignment_loss‚3.129420137986904 1loss/SRqqt3/assignment_loss‚1.071388105869234 1loss/FBt4/assignment_loss‚0.36393337137599246 epoch‚9 _runtime‚9251.408608973 /loss/FRt1/assignment_loss‚3.317117997578212 /loss/FRt4/detection_loss‚2.3514668112810777 1loss/SRqqt2/detection_loss‚1.8173722826772265 0loss/SRqqt3/detection_loss‚1.124687996175554 /loss/FBt4/detection_loss‚3.6247353553771973 1loss/FBt2/assignment_loss‚0.29835799450258527 /loss/FRt3/detection_loss‚1.9371051874784928 0loss/SRqqt1/detection_loss‚2.538599807640602 1loss/SRqqt4/assignment_loss‚1.292714363662526 0loss/FBt3/assignment_loss‚0.1853695550758361ºÈœÉ“ö)‚(:ˆ ûÉæÔÉø…ý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚185.164' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.0328 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0jÃçyr÷)‚(:ç ûÉæÔã²÷ÿ network.recv‚ 3.294436e+06 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚37407.6689453125 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 disk.dm-1.out‚ 360.109375 proc.cpu.threads‚17* disk./.usagePercent‚20.434186781212766 disk.sda1.out‚0 disk.drbd1716.out‚0) proc.memory.availableMB‚ 1002.72265625 network.sent‚ 2.8540605e+07 disk.sda3.out‚ 360.109375" disk.dm-0.out‚37407.6689453125$ memory_percent‚93.87986660003662 cpu‚25.34217588052515 disk.drbd2266.out‚0 disk.rbd1.out‚ 56.046875 disk.drbd2265.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚0.078125 disk.drbd2265.out‚0$ disk./.usageGB‚304.4841499328613 disk.sda.out‚ 360.109375 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0* proc.memory.percent‚15.674829483032227 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.sda2.in‚0% disk.nvme0n1.out‚37407.6689453125 disk.dm-1.in‚ 0.01171875" proc.memory.rssMB‚ 2568.1640625{Ž%³žø)‚(:“ ŠÊæÔÒ–™ý gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚135.311' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚27.062200000000004 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0öoéqù)‚(:æ ŠÊæÔ£ú„ÿ disk.rbd1.out‚ 56.046875 disk.drbd2266.in‚0 disk.rbd0.out‚0.078125 disk.sda1.out‚0) proc.memory.availableMB‚ 1002.72265625 disk.dm-0.in‚0.078125 disk.drbd2265.out‚0 disk.rbd1.in‚0 disk.dm-1.out‚ 360.71875 disk.nvme0c0n1.in‚0.078125' disk.nvme0c0n1.out‚37447.8720703125" proc.memory.rssMB‚ 2568.1640625* proc.memory.percent‚15.674829483032227% disk./.usageGB‚304.49544525146484 disk.drbd2266.out‚0 disk.sda.out‚ 360.71875 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd1716.out‚0 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚37447.8720703125" disk.dm-0.out‚37447.8720703125 disk.sda.in‚ 0.01171875 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.drbd2265.in‚0 network.sent‚ 2.8650067e+07* disk./.usagePercent‚20.434944819521654 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 360.71875 network.recv‚ 3.301398e+06 disk.drbd2080.out‚0 disk.sda1.in‚0$ memory_percent‚93.87986660003662 cpu‚25.341682669478097›€Wf‘ú)‚(:† ™ÊæÔÓ¡“ý gpu.0.gpu‚43.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚133.13' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚26.626 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‡IÖsû)‚(:è ™ÊæÔ´´‘€ network.recv‚ 3.304974e+06$ disk./.usageGB‚304.4897270202637 disk.nvme0n1.in‚0.078125 disk.drbd1716.in‚0 disk.drbd2265.out‚0 disk.rbd1.in‚0* disk./.usagePercent‚20.434561064227825 disk.drbd2265.in‚0 disk.drbd2080.in‚0" proc.memory.rssMB‚ 2568.1640625 proc.cpu.threads‚17 disk.drbd2080.out‚0' disk.nvme0c0n1.out‚37488.8720703125 disk.drbd2266.out‚0 disk.rbd0.in‚0" disk.dm-0.out‚37488.8720703125 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0 disk.rbd1.out‚ 56.1953125 disk.sda.in‚ 0.01171875 cpu‚25.34120674886105% disk.nvme0n1.out‚37488.8720703125 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 361.359375 disk.sda2.in‚0 network.sent‚ 2.8655343e+07 disk.dm-1.out‚ 361.359375 disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda1.out‚0) proc.memory.availableMB‚ 1002.48828125 disk.rbd0.out‚0.078125 disk.sda.out‚ 361.359375 disk.sda2.out‚0$ memory_percent‚93.88129711151123* proc.memory.percent‚15.674829483032227e‰£«2ü)- _wandbruntime‚9288  _runtime‚9288M¦¹$Èý) & lr-AdamW/pg1‚5.127670006816001e-07 & lr-AdamW/pg2‚5.127670006816001e-07 trainer/global_step‚11649 % _timestamp‚1.7884541732129054e+09 _runtime‚9288.684557596 _step‚955­ÜîÍþ)Ç trainer/global_step‚11649 % _timestamp‚1.7884541732129054e+09 _runtime‚9288.684557596 _step‚955 & lr-AdamW/pg1‚5.127670006816001e-07 & lr-AdamW/pg2‚5.127670006816001e-07»þ:á2ÿ)- _wandbruntime‚9288  _runtime‚9288WLX ê'€*äO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 219922197 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚677412423391744 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 157332480 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚659660366042112 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚669381740010378 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 169420998 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚687169719328768 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚55755208 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚687169044963328 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 219922197 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚458 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _step‚956 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9756621571584 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚11649 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 154605878 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚659660383081472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚677412423391744 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚55754752 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9757277934080 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9757277934080 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚687169719328768 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9756621571584 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚739727289993728 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9720702491150 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚677412441394688 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 219942235 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 389363235 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 219942235 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 154605420 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10238540109312 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚9288.685509613 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 389343195 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚749965806162944 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚98850668 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚687169044963328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚98850670 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9757277934080 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9757277934080 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚687169719328768 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9756621571584 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚739727289993728 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9720702491150 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚677412441394688 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 219942235 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 389363235 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 219942235 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 154605420 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10238540109312 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _runtime‚9288.685509613 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 389343195 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚749965806162944 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚98850668 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚687169044963328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 181272576 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚98850670 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚687169044963328 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 219922197 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚458 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 _step‚956 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9756621571584 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 trainer/global_step‚11649 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 154605878 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚659660383081472¼Ö-¶2‚*- _wandbruntime‚9289  _runtime‚9289¯tl„¡ ƒ*îM @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9758097375232 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 169435014 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 % _timestamp‚1.7884541739398723e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9757440999424 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9721518926114 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚669437460576110 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment[„rCS.large_pool.peak‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚677468794433536 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚739788916791296 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚98858838 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9722173364078 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚687226891808768 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚677468794433536 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 154618134 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚55759750 _step‚957 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚739788940731392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚687226891808768 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55759298 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚659715287212032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚677468776430592 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚659715270172672 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 389395731 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 219940651 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚98858836 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 389375665 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 219960715 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 389375665 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚669436789098786 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 219960715 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10239399464960 trainer/global_step‚11649 _runtime‚9289.411884847 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9757440999424 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 169435014 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 389395731 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚750028316256256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 169435016 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚677468776430592 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9758097375232 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 154618588 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9758097375232 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 169435014 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 % _timestamp‚1.7884541739398723e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9757440999424 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9721518926114 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚669437460576110 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚677468794433536 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚739788916791296 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚98858838 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9722173364078 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚687226891808768 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚677468794433536 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 154618134 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚55759750 _step‚957 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚739788940731392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚687226891808768 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55759298 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚659715287212032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚677468776430592 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚659715270172672 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 389395731 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 219940651 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚98858836 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 389375665 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 219960715 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 389375665 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚454 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚669436789098786 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 219960715 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚452 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10239399464960 trainer/global_step‚11649 _runtime‚9289.411884847 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9757440999424 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 169435014 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 389395731 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚750028316256256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 169435016 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚677468776430592 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9758097375232 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 154618588 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚672167781877786 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚9327.549192258 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚753091696488960 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9798263739904 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚680230093337088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9798263739904 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 170121800 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚753091500536320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚690027682711552 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚680230075334144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚662405572570112 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚99259068 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 220848249 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚99259070 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚960 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚459 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9797607377408 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9761537830558 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 % _timestamp‚1.7884542120771832e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚690027682711552 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 155242602 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 390970047 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚55983534 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 170121798 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚742809826876928 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚672167781877786 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚9327.549192258 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚753091696488960 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9798263739904 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚680230093337088 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9798263739904 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 170121800 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚753091500536320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚690027682711552 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚680230075334144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚662405572570112 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚99259068 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 172012544 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 220848249 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚99259070 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚960 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚459 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9797607377408 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9761537830558 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 % _timestamp‚1.7884542120771832e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚690027682711552 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 155242602 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 390970047 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚55983534 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 170121798 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚742809826876928 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 195952640 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9798426805248 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9798426805248 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚680286428372992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚662460493740032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚961 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚690084855178240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 391022583 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 155255330 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55988094 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9763008703486 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 170135816 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚690085529556992 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 391002517 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10282725099008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9762354265522 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9799083181056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚662460476700672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚474 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚99267238 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 155255804 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚472 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 _runtime‚ 9328.28051726 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚680286446375936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚742871477614592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 391022583 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844ásï÷&”*ñM MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 183356416 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚680286446375936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚742871477614592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 391022583 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 $ _timestamp‚1.788454212808348e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚742871453674496 trainer/global_step‚11699 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚672222830966194 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚680286428372992 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 220886767 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚55988566 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚753154019357184 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 220866703 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚753154202713600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9798426805248 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9798426805248 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚680286428372992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚662460493740032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _step‚961 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚690084855178240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 391022583 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 155255330 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚55988094 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9763008703486 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 170135816 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚690085529556992 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 391002517 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10282725099008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9762354265522 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9799083181056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚662460476700672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚474 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚99267238 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 155255804 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚472 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 _runtime‚ 9328.28051726 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0ÁöÕŸ 2•*- _wandbruntime‚9328  _runtime‚9328zžª©S–*Í /loss/FBt3/detection_loss‚0.5709265936915388 /loss/FBt4/detection_loss‚3.3214662273724875 % _timestamp‚1.7884542128101153e+09 /loss/FRt1/assignment_loss‚4.103108863035838 epoch‚9 _step‚962 &loss/total_loss‚44.405506306394024 0loss/FBt4/assignment_loss‚0.3397606735428174 /loss/FRt2/assignment_loss‚3.558061387087848 /loss/FRt4/detection_loss‚2.1385777367485894 1loss/SRqqt2/assignment_loss‚1.285410073896249 .loss/FRt2/detection_loss‚2.218914638499956 1loss/SRqqt3/detection_loss‚1.1543243357290824 1loss/SRqqt4/detection_loss‚1.4693466603474796 trainer/global_step‚11699 1loss/SRqqt4/assignment_loss‚1.508969893292436 0loss/FBt3/assignment_loss‚0.2707256295137347 .loss/FRt3/detection_loss‚1.674692469217849 0loss/SRqqt1/detection_loss‚2.485046309232712 _runtime‚9328.281962067 /loss/FBt1/detection_loss‚2.8851816654205322 .loss/FBt2/detection_loss‚2.859552842875322 .loss/FRt1/detection_loss‚3.126902704437574 /loss/FRt3/assignment_loss‚1.383981865865213 2loss/SRqqt3/assignment_loss‚1.0241800408034274 0loss/FBt1/assignment_loss‚0.5375911262297248 /loss/FBt2/assignment_loss‚0.316145963811626 .loss/FRt4/assignment_loss‚2.49205801486969 2loss/SRqqt1/assignment_loss‚1.6738192293792964 0loss/SRqqt2/detection_loss‚2.006761361493005m¿™—*Ò /loss/FBt2/assignment_loss‚0.316145963811626 .loss/FRt4/assignment_loss‚2.49205801486969 2loss/SRqqt1/assignment_loss‚1.6738192293792964 0loss/SRqqt2/detection_loss‚2.006761361493005 /loss/FBt3/detection_loss‚0.5709265936915388 /loss/FBt4/detection_loss‚3.3214662273724875 % _timestamp‚1.7884542128101153e+09 /loss/FRt1/assignment_loss‚4.103108863035838 epoch‚9 _step‚962 &loss/total_loss‚44.405506306394024 0loss/FBt4/assignment_loss‚0.3397606735428174 /loss/FRt2/assignment_loss‚3.558061387087848 /loss/FRt4/detection_loss‚2.1385777367485894 1loss/SRqqt2/assignment_loss‚1.285410073896249 .los‹ jv¿s/FRt2/detection_loss‚2.218914638499956 1loss/SRqqt3/detection_loss‚1.1543243357290824 1loss/SRqqt4/detection_loss‚1.4693466603474796 trainer/global_step‚11699 1loss/SRqqt4/assignment_loss‚1.508969893292436 0loss/FBt3/assignment_loss‚0.2707256295137347 .loss/FRt3/detection_loss‚1.674692469217849 0loss/SRqqt1/detection_loss‚2.485046309232712 _runtime‚9328.281962067 /loss/FBt1/detection_loss‚2.8851816654205322 .loss/FBt2/detection_loss‚2.859552842875322 .loss/FRt1/detection_loss‚3.126902704437574 /loss/FRt3/assignment_loss‚1.383981865865213 2loss/SRqqt3/assignment_loss‚1.0241800408034274 0loss/FBt1/assignment_loss‚0.5375911262297248Âõ« Sž˜*‚(:“ ÆÊæÔ„§ý gpu.0.gpu‚51.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚144.545' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚28.908999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ôHjz™*‚(:ï ÆÊæÔü•Àÿ disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.sda3.in‚ 0.01171875* proc.memory.percent‚15.674829483032227 network.sent‚ 2.8776231e+07* disk./.usagePercent‚20.453377874002022 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚0.078125 disk.drbd2266.out‚0 cpu‚25.339810107697758 disk.sda1.out‚0& disk.nvme0n1.out‚37595.54345703125( disk.nvme0c0n1.out‚37595.54345703125# disk.dm-0.out‚37595.54345703125 network.recv‚ 3.319952e+06 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 363.06640625 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.sda1.in‚0 disk.sda2.out‚0 disk.sda3.out‚ 363.06640625 disk.sda.out‚ 363.06640625 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.rbd1.out‚57.03125 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.dm-0.in‚0.078125$ memory_percent‚93.88260841369629" proc.memory.rssMB‚ 2568.1640625 disk.sda2.in‚0 disk.nvme0n1.in‚0.078125$ disk./.usageGB‚304.7701110839844 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875( proc.memory.availableMB‚ 1002.2734375’/sžš*‚(:“ ÕÊæÔ†Žþ gpu.0.gpu‚47.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚181.168' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚36.233599999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0µ‡{y›*‚(:î ÕÊæÔåö—ÿ disk.sda.in‚ 0.01171875 disk.sda.out‚ 363.73046875" proc.memory.rssMB‚ 2568.1640625% disk./.usageGB‚304.83584213256836 disk.nvme0n1.in‚0.078125% disk.nvme0n1.out‚37643.5771484375$ memory_percent‚93.88258457183838) proc.memory.availableMB‚ 1002.27734375" disk.dm-0.out‚37643.5771484375 disk.dm-1.out‚ 363.73046875 disk.drbd1716.out‚0 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0.078125 disk.dm-0.in‚0.078125 network.sent‚ 2.8884085e+07 network.recv‚ 3.325269e+06 disk.drbd1716.in‚0 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 363.73046875 disk.rbd1.out‚57.53125 disk.drbd2080.in‚0 disk.rbd0.in‚0 disk.sda1.in‚0* proc.memory.percent‚15.674829483032227 disk.drbd2265.out‚0 proc.cpu.threads‚17 disk.sda1.out‚0' disk.nvme0c0n1.out‚37643.5771484375 disk.sda3.in‚ 0.01171875 disk.sda2.in‚0 disk.sda2.out‚0* disk./.usagePercent‚20.457789139824513 disk.dm-1.in‚ 0.01171875 disk.rbd1.in‚0 cpu‚25.339374825598572 disk.drbd2266.out‚0|£;‘œ*‚(:† äÊæÔÍ›œý gpu.0.gpu‚21.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚181.91' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.382 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ŠaK z*‚(:ï äÊæÔû‚ÿþ# disk./.usageGB‚304.871639251709 disk.nvme0c0n1.in‚0.078125 disk.sda3.out‚ 364.21484375 cpu‚25.338887035841807 proc.cpu.threads‚17* disk./.usagePercent‚20.460191514525828 disk.sda1.in‚0 disk.sda2.out‚0 network.recv‚ 3.330105e+06 disk.dm-0.in‚0.078125 disk.sda.out‚ 364.21484375 disk.rbd1.in‚0 disk.rbd1.out‚ 58.73046875 disk.nvme0n1.in‚0.078125) proc.memory.availableMB‚ 1002.28515625 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.sda2.in‚0" proc.memory.rssMB‚ 2568.1640625* proc.memory.percent‚15.674829483032227 network.sent‚ 2.8890531e+07 disk.drbd2265.in‚0$ memory_percent‚93.88253688812256 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 364.21484375 disk.drbd2265.out‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0.078125 disk.sda1.out‚0' disk.nvme0c0n1.out‚37681.2412109375" disk.dm-0.out‚37681.2412109375% disk.nvme0n1.out‚37681.24121093753t”^2ž*- _wandbruntime‚9365  _runtime‚9365Äx²½ÊŸ*Ä ' lr-AdamW/pg1‚3.0675215478622455e-07 ' lr-AdamW/pg2‚3.0675215478622455e-07 trainer/global_step‚11749 % _timestamp‚1.7884542503886616e+09 _runtime‚9365.860330196 _step‚963àzn|Ï *É _runtime‚9365.860330196 _step‚963 ' lr-AdamW/pg1‚3.0675215478622455e-07 ' lr-AdamW/pg2‚3.0675215478622455e-07 trainer/global_step‚11749 % _timestamp‚1.7884542503886616e+09ú¨—2¡*- _wandbruntime‚9365  _runtime‚9365i ?é'¢*ãO O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 221774301 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9839249545728 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 155880276 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚674953152268078 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9838593183232 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9802373169966 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 392596899 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚745892339820032 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9803027607722 trainer/global_step‚11749 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 170822598 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 221794339 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚745892363760128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚692886320459776 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚692886994825216 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚448 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚756217104289792 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚683047745279488 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9838593183232 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚692886320459776 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚692886994825216 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚448 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚756217104289792 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚683047745279488 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9838593183232 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚674953152268078 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9838593183232 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 159429632 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9802373169966 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 392596899 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚745892339820032 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9803027607722 trainer/global_step‚11749 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 170822598 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 221794339 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚745892363760128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚692944167305216 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 392649435 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚756279808691200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 170836616 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚745954014497792 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚675009544310926 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚665205700268032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚675008872833602 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚683104080315392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚56217411 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚745953990557696 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 221792755 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9839412611072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚692943492926464 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 170836616 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 221812819 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 155893496 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 155893047 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 170836614 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚683104080315392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚56217858 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚692944167305216 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 392649435 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9839412611072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚99675636 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 392629369 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 65637[Úêæ\5808 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚9366.582220902 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884542511101449e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9840068986880 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9803189604930 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚11749 _step‚965 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9840068986880 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚99675638 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚665205683228672 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9803844042894 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚756279631626240èF†‹ù&¦*óM OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚447 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚683104098318336 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 392629369 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 170836614 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 % _timestamp‚1.7884542511101449e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9840068986880 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9803189604930 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚11749 _step‚965 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9840068986880 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚99675638 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚665205683228672 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9803844042894 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚756279631626240 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚692944167305216 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 392649435 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚756279808691200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 170836616 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚449 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚745954014497792 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 153124864 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚675009544310926 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 177064960 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚665205700268032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚675008872833602 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚683104080315392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚56217411 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚745953990557696 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 221792755 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9839412611072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚692943492926464 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 170836616 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 221812819 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 155893496 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 155893047 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 170836614 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚683104080315392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚56217858 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚692944167305216 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 392649435 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9839412611072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚99675636 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 392629369 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _runtime‚9366.582220902 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚685865379218944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚667895985626112 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9880235351552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9880235351552 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9843862947130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 222720391 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 171523398 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚748974900643328 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚748974876703232 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚11799 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚464 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10368120832000 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 156517758 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚667896002665472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9843208509374 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 222700353 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚695744958208000 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚685865397221888 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚677739865612602 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚695744958208000 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚685865397221888 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9879578989056 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 174109696 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 394223751 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚466 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚56441888 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚ 100075868 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 171523400 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚685865379218944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚667895985626112 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9880235351552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 9880235351552 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9843862947130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 222720391 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 171523398 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚748974900643328 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚748974876703232 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚11799 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚464 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10368120832000 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 156517758 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚667896002665472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9843208509374 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 222700353 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚695744958208000 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚685865397221888 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚677739865612602 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚695744958208000 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚685865397221888 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9879578989056 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 174109696 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 394223751 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚466 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚56441888 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚ 100075868 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 171523400 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 222718807 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚685921750260736 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 156530003 _step‚969 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 171537416 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 156530458 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚749036527440896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚677795586178334 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9844024944338 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 394256221 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚685921750260736 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9881054792704 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 222718807 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚677794914701010 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 394276287 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚685921732257792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚695802130674688 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚685921732257792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚56445967 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 171537416 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚ 100084036 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚759405527989760 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10368976608768 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10368817192448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚667950906796032 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9844679382302 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 394256221 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9881054792704 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚695802805053440 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9880398416896 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884542890354795e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 394276287 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455[ÕÏû&¶*õM JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚685921750260736 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9881054792704 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 222718807 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚677794914701010 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 394276287 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚685921732257792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚695802130674688 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚685921732257792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚56445967 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 159416320 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 171537416 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚ 100084036 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚759405527989760 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10368976608768 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10368817192448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚667950906796032 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9844679382302 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 394256221 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 9881054792704 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚695802805053440 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9880398416896 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 % _timestamp‚1.7884542890354795e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 394276287 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚455 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9880398416896 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 222738871 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 100084038 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚56446420 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚749036551380992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚759405344633344 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 222718807 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚685921750260736 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 156530003 _step‚969 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 171537416 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 156530458 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚749036527440896 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚677795586178334 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9844024944338 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 394256221ÉÐ ´H2·*- _wandbruntime‚9404  _runtime‚9404Ø'FwY¸*Ó _runtime‚9404.508611035 2loss/SRqqt3/assignment_loss‚0.9116703963591135 2loss/SRqqt4/assignment_loss‚1.1989129823886535 /loss/FRt2/detection_loss‚2.3925208173692227 .loss/FBt4/detection_loss‚3.543250262737274 2loss/SRqqt1/assignment_loss‚1.4685121986728449 2loss/SRqqt2/assignment_loss‚1.6802875946462155 /loss/FBt2/detection_loss‚1.9132711431559395 _step‚970 0loss/FBt3/assignment_loss‚0.2199868567929392 /loss/FRt1/detection_loss‚2.8382725045084953 trainer/global_step‚11799 0loss/FRt3/assignment_loss‚1.3499703013896942 /loss/FBt2/assignment_loss‚0.537495270261868 0loss/SRqqt1/detection_loss‚2.956111476971553 0loss/SRqqt3/detection_loss‚1.007308449683478 /loss/FBt1/detection_loss‚2.3541264639181247 epoch‚9 0loss/FRt1/assignment_loss‚3.5298733972012997 1loss/FBt1/assignment_loss‚0.39128419619159743 .loss/FRt4/detection_loss‚2.185347337126732 $ _timestamp‚1.788454289037108e+09 .loss/FRt3/detection_loss‚1.812660580575466 0loss/SRqqt2/detection_loss‚1.888177098631859 1loss/SRqqt4/detection_loss‚1.8068507606332953 /loss/FBt3/detection_loss‚0.5951022267023601 0loss/FRt2/assignment_loss‚3.6334552586078646 0loss/FRt4/assignment_loss‚2.4923850333690645 0loss/FBt4/assignment_loss‚0.6819934891730858 %loss/total_loss‚43.38882609706805¼Áß^¹*Ø 0loss/FRt2/assignment_loss‚3.6334552586078646 0loss/FRt4/assignment_loss‚2.4923850333690645 0loss/FBt4/assignment_loss‚0.6819934891730858 %loss/total_loss‚43.38882609706805 _runtime‚9404.508611035 2loss/SRqqt3/assignment_loss‚0.9116703963591135 2loss/SRqqt4/assignment_loss‚1.1989129823886535 /loss/FRt2/detection_loss‚2.3925208173692227 .loss/FBt4/detection_loss‚3.543250262737274 2loss/SRqqt1/assignment_loss‚1.4685121986728449 2loss/SRqqt2/assignment_loss‚1.6802875946462155 /loss/FBt2/detection_loss‚1.9132711431559395 _step‚970 0loss/FBt3/assignment_loss‚0.2199868567929392 /loss/FRt1/detection_loss‚2.8382725045084953 trainer/global_step‚11799 0loss/FRt3/assignment_loss‚1.3499703013896942 /loss/FBt2/assignment_loss‚0.537495270261868 0loss/SRqqt1/detection_loss‚2.956111476971553 0loss/SRqqt3/detection_loss‚1.007308449683478 /loss/FBt1/detection_loss‚2.3541264639181247 epoch‚9 0loss/FRt1/assignment_loss‚3.5298733972012997 1loss/FBt1/assignment_loss‚0.39128419619159743 .loss/FRt4/detection_loss‚2.185347337126732 $ _timestamp‚1.788454289037108e+09 .loss/FRt3/detection_loss‚1.812660580575466 0loss/SRqqt2/detection_loss‚1.888177098631859 1loss/SRqqt4/detection_loss‚1.8068507606332953 /loss/FBt3/detection_loss‚0.5951022267023601Ê{¯½º*‚(:’ ‘ËæÔõ»•ý gpu.0.gpu‚48.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚171.91' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚34.382000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÊDEx»*‚(:í ‘ËæÔÐÕ„ÿ disk.drbd2266.out‚0 disk.dm-0.in‚0.078125 disk.sda1.in‚0 disk.sda2.in‚0 disk.dm-1.out‚ 366.28515625' proc.memory.availableMB‚ 1002.703125 proc.cpu.threads‚17 network.recv‚ 3.343106e+06 disk.drbd2265.in‚0' disk.nvme0c0n1.out‚37787.1572265625 disk.sda3.out‚ 366.28515625 disk.rbd0.in‚0" disk.dm-0.out‚37787.1572265625 disk.drbd2080.in‚0 disk.drbd2266.in‚0 disk.sda.out‚ 366.28515625% disk.nvme0n1.out‚37787.1572265625% disk./.usageGB‚302.61055755615234 disk.drbd1716.in‚0 disk.nvme0c0n1.in‚0.078125* proc.memory.percent‚15.674829483032227 cpu‚25.33753529261235) disk./.usagePercent‚20.30844842476311 disk.drbd2080.out‚0 disk.sda1.out‚0 disk.nvme0n1.in‚0.078125$ memory_percent‚93.87998580932617 disk.rbd1.out‚ 59.81640625 network.sent‚ 2.9009276e+07 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.sda2.out‚0 disk.dm-1.in‚ 0.01171875 disk.rbd0.out‚0.078125 disk.sda.in‚ 0.01171875 disk.rbd1.in‚0" proc.memory.rssMB‚ 2568.1640625 disk.drbd1716.out‚0”]´“¼*‚(:ˆ  ËæÔ‘Ó»ý gpu.0.gpu‚55.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚125.839' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚25.1678 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0—Ák½*‚(:ê  ËæÔµ÷É€ disk.nvme0c0n1.in‚0.796875 network.recv‚ 3.349781e+06" disk.dm-0.out‚uÉŒ0 37899.1650390625 disk.sda1.in‚0 disk.rbd1.out‚ 59.81640625 disk.dm-1.out‚ 366.671875 disk.drbd1716.in‚0 disk.sda3.out‚ 366.671875 network.sent‚ 2.9118276e+07$ disk./.usageGB‚302.8600540161133 disk.sda2.in‚0 disk.nvme0n1.in‚0.796875% disk.nvme0n1.out‚37899.1650390625 disk.dm-1.in‚ 0.01171875 cpu‚25.337017372236254 disk.rbd0.in‚0 disk.sda2.out‚0" proc.memory.rssMB‚ 2568.1640625 disk.dm-0.in‚0.796875 disk.sda.out‚ 366.671875 disk.drbd1716.out‚0) proc.memory.availableMB‚ 1002.64453125' disk.nvme0c0n1.out‚37899.1650390625 disk.sda.in‚ 0.01171875 disk.drbd2265.in‚0 disk.drbd2080.in‚0 disk.rbd1.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875* proc.memory.percent‚15.674829483032227 disk.rbd0.out‚0.078125 disk.sda1.out‚0 disk.drbd2080.out‚0* disk./.usagePercent‚20.325192341532567 disk.drbd2266.in‚0 disk.drbd2266.out‚0$ memory_percent‚93.88034343719482 proc.cpu.threads‚17]ìoÝ“¾*‚(:ˆ ¯ËæÔÄšý gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚180.013' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.0026 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0_²s?w¿*‚(:ì ¯ËæÔ굯ÿ) disk./.usagePercent‚20.34203840530818 disk.drbd2266.in‚0 disk.sda2.in‚0$ memory_percent‚93.88048648834229 disk.nvme0n1.in‚1.265625 disk.sda1.in‚0 network.recv‚ 3.35283e+06 disk.dm-1.out‚ 367.37890625 disk.sda1.out‚0 disk.sda2.out‚0" proc.memory.rssMB‚ 2568.1640625 proc.cpu.threads‚17 disk.dm-0.in‚1.265625 disk.sda.in‚ 0.01171875 disk.sda.out‚ 367.37890625 disk.drbd2265.out‚0$ disk./.usageGB‚303.1110725402832 disk.drbd1716.out‚0 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.rbd1.in‚0 disk.drbd1716.in‚0 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 367.37890625! disk.dm-0.out‚38010.876953125* proc.memory.percent‚15.674829483032227 network.sent‚ 2.9123434e+07 disk.rbd1.out‚ 59.9814453125 disk.rbd0.out‚0.078125 cpu‚25.33656914785477& disk.nvme0c0n1.out‚38010.876953125 disk.rbd0.in‚0) proc.memory.availableMB‚ 1002.62109375$ disk.nvme0n1.out‚38010.876953125 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚1.265625 disk.drbd2266.out‚0%?2À*- _wandbruntime‚9440  _runtime‚9440MÒw·ÊÁ*Ä ' lr-AdamW/pg1‚1.5328108145847245e-07 ' lr-AdamW/pg2‚1.5328108145847245e-07 trainer/global_step‚11849 % _timestamp‚1.7884543247924688e+09 _runtime‚9440.264071561 _step‚971wŒÄçÏÂ*É ' lr-AdamW/pg1‚1.5328108145847245e-07 ' lr-AdamW/pg2‚1.5328108145847245e-07 trainer/global_step‚11849 % _timestamp‚1.7884543247924688e+09 _runtime‚9440.264071561 _step‚971ËQHZ2Ã*- _wandbruntime‚9440  _runtime‚9440ÂÃ’%ì'Ä*æO ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚670641209193472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9884043848782 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9884698286538 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 223626405 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10411076624896 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚688683049164288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚680525907480010 _step‚972 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 395850603 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚698603595956224 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9920564794880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 trainer/global_step‚11849 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 172224198 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 157156072 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9921221157376 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10411236054528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 395870643 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚56672240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚56671804 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚688683031161344 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚680525236002894 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚ 100484268 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 157156510 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚698604270321664 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚698604270321664 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9920564794880 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚9440.265120691 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 223646443 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 172224200 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 172224198 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚698603595956224 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 172224200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040~jßñ'Å*ëO H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚670641209193472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9884043848782 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 9884698286538 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 223626405 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10411076624896 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚688683049164288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚438 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚680525907480010 _step‚972 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 395850603 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚698603595956224 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 9920564794880 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 trainer/global_step‚11849 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚20038 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 172224198 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 157156072 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9921221157376 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10411236054528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 395870643 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚56672240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚56671804 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚688683031161344 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚680525236002894 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 UGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚ 100484268 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 157156510 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 339635712 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚698604270321664 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚698604270321664 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 9920564794880 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚436 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚9440.265120691 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚ 223646443 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚ 172224200 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 172224198 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚698603595956224 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 172224200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414ÌÍÑ$…2Æ*- _wandbruntime‚9440  _runtime‚94405?ʧ“#Ç*ðM H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 172238214 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 157169205 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9884860283746 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 395883073 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 172238216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 395903139 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚688739402203136 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 395903139 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 157168769 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚752119088264192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 223664923 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 223644859 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚56676767 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 100492438 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚698660768422912 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 223644859 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 172238214 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚56676333 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚752119064324096 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9921384222720 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚670696113324032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 395883073 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_p±‹ çcool.current‚434 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 223664923 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0D€^‘û&È*õM QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 172238216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 395903139 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚688739402203136 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 395903139 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 181259264 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 157168769 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚752119088264192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 223664923 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 223644859 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚56676767 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚24401 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 100492438 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚698660768422912 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 223644859 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 172238214 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚56676333 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚752119064324096 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚844 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 9921384222720 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚670696113324032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 395883073 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚434 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 223664923 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚436 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 172238214 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 157169205 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9884860283746 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 395883073Í(}©>2É*- _wandbruntime‚9440  _runtime‚9440Çʨ_Ê*Ù 0loss/FRt4/assignment_loss‚2.9266266614198684 /loss/FBt4/detection_loss‚2.8537037670612335 0loss/FRt1/assignment_loss‚3.7983241427329277 /loss/FRt2/assignment_loss‚3.116663224167294 2loss/SRqqt2/assignment_loss‚1.6672349736954157 1loss/FBt3/assignment_loss‚0.23927847225253535 /loss/FBt2/detection_loss‚2.4840290397405624 _runtime‚9440.945941461 2loss/SRqqt4/assignment_loss‚1.7380054676141896 1loss/FBt1/assignment_loss‚0.45375494098608254 .loss/FRt1/detection_loss‚2.994583814374862 1loss/SRqqt2/detection_loss‚1.6378446576687007 /loss/SRqqt4/detection_loss‚1.46307378921841 &loss/total_loss‚43.638799417590704 2loss/SRqqt3/assignment_loss‚1.1530224855989217 1loss/SRqqt3/detection_loss‚0.9964154861370722 % _timestamp‚1.7884543254744494e+09 /loss/FRt4/detection_loss‚2.6666053773835303 0loss/SRqqt1/detection_loss‚2.697741375815484 2loss/SRqqt1/assignment_loss‚1.9865563694507844 1loss/FBt2/assignment_loss‚0.44299056389718316 .loss/FRt3/detection_loss‚1.643266776404171 0loss/FRt3/assignment_loss‚1.4025517944936399 /loss/FRt2/detection_loss‚2.0617559817102222 /loss/FBt3/detection_loss‚0.6511843551048323 epoch‚9 trainer/global_step‚11849 _step‚974 1loss/FBt4/assignment_loss‚0.15338139573577791 /loss/FBt1/detection_loss‚2.4102045049269996Œð cdË*Þ 1loss/SRqqt2/detection_loss‚1.6378446576687007 /loss/SRqqt4/detection_loss‚1.46307378921841 &loss/total_loss‚43.638799417590704 2loss/SRqqt3/assignment_loss‚1.1530224855989217 1loss/SRqqt3/detection_loss‚0.9964154861370722 % _timestamp‚1.7884543254744494e+09 /loss/FRt4/detection_loss‚2.6666053773835303 0loss/SRqqt1/detection_loss‚2.697741375815484 2loss/SRqqt1/assignment_loss‚1.9865563694507844 1loss/FBt2/assignment_loss‚0.44299056389718316 .loss/FRt3/detection_loss‚1.643266776404171 0loss/FRt3/assignment_loss‚1.4025517944936399 /loss/FRt2/detection_loss‚2.0617559817102222 /loss/FBt3/detection_loss‚0.6511843551048323 epoch‚9 trainer/global_step‚11849 _step‚974 1loss/FBt4/assignment_loss‚0.15338139573577791 /loss/FBt1/detection_loss‚2.4102045049269996 0loss/FRt4/assignment_loss‚2.9266266614198684 /loss/FBt4/detection_loss‚2.8537037670612335 0loss/FRt1/assignment_loss‚3.7983241427329277 /loss/FRt2/assignment_loss‚3.116663224167294 2loss/SRqqt2/assignment_loss‚1.6672349736954157 1loss/FBt3/assignment_loss‚0.23927847225253535 /loss/FBt2/detection_loss‚2.4840290397405624 _runtime‚9440.945941461 2loss/SRqqt4/assignment_loss‚1.7380054676141896 1loss/FBt1/assignment_loss‚0.45375494098608254 .loss/FRt1/detection_loss‚2.994583814374862Îô.%“Ì*‚(:ˆ ¾ËæÔ²€‰ý gpu.0.gpu‚34.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚124.038' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.8076 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÓduÍ*‚(:ê ¾ËæÔ•¤Áÿ disk.rbd0.in‚0 disk.dm-0.in‚ 1.7421875 disk.dm-1.out‚ 368.171875 disk.drbd2265.out‚0 cpu‚25.336062155621377) disk./.usagePercent‚20.37564272227516 disk.sda3.out‚ 368.171875 disk.sda.in‚ 0.01171875 disk.sda1.out‚0 disk.drbd2080.in‚0) proc.memory.availableMB‚ 1002.34765625 proc.cpu.threads‚17 disk.drbd2266.in‚0 disk.sda.out‚ 368.171875 disk.sda2.in‚0 disk.nvme0n1.in‚ 1.7421875$ disk.nvme0n1.out‚38257.642578125 disk.drbd2265.in‚0" memory_percent‚93.882155418396 disk.rbd1.in‚0 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125 disk.rbd1.out‚ 59.9814453125& disk.nvme0c0n1.out‚38257.642578125 network.sent‚ 2.9130171e+07 disk.sda3.in‚ 0.01171875 disk.sda1.in‚0 disk.sda2.out‚0 disk.drbd1716.in‚0 disk.drbd1716.out‚0" proc.memory.rssMB‚ 2568.1640625! disk.dm-0.out‚38257.642578125 network.recv‚ 3.357826e+06% disk./.usageGB‚303.61180114746094 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚ 1.7421875* proc.memory.percent‚15.6748294830322276Ô˜“Î*‚(:ˆ ÍËæÔÜóü gpu.0.gpu‚40.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚181.082' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚36.2164 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ÇžÏ*‚(:ô ÍËæÔÛøÿþ* disk./.usagePercent‚20.375508830334756 disk.sda1.out‚0 disk.sda2.in‚0 disk.drbd2265.in‚0 disk.rbd0.out‚0.078125 disk.sda1.in‚0 disk.nvme0n1.in‚2.203125 disk.nvme0c0n1.in‚2.203125 network.recv‚ 3.363339e+06 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 369.05078125 disk.drbd2266.out‚0 disk.sda3.out‚ 369.05078125 cpu‚25.335560397387432# disk./.usageGB‚303.609806060791 disk.dm-0.in‚2.203125 disk.drbd1716.in‚0 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.drbd2080.out‚0$ memory_percent‚93.88067722320557* proc.memory.percent‚15.674829483032227 network.sent‚ 2.9237917e+07 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 disk.sda.out‚ 369.05078125 disk.rbd1.in‚0 proc.cpu.threads‚17 disk.rbd0.in‚0# disk.dm-0.out‚38549.64697265625 disk.rbd1.out‚ 60.0439453125& disk.nvme0n1.out‚38549.64697265625( disk.nvme0c0n1.out‚38549.64697265625) proc.memory.availableMB‚ 1002.58984375 disk.sda2.out‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875" proc.memory.rssMB‚ 2568.1640625Hº§2Ð*- _wandbruntime‚9477  _runtime‚9477“ß"5ÈÑ* & lr-AdamW/pg1‚9.629115037250902e-08 & lr-AdamW/pg2‚9.629115037250902e-08 trainer/global_step‚11899 % _timestamp‚1.7884543615702515e+09 _runtime‚9477.041852298 _step‚975šKÍÒ*Ç & lr-AdamW/pg2‚9.629115037250902e-08 trainer/global_step‚11899 % _timestamp‚1.7884543615702515e+09 _runtime‚9477.041852298 _step‚975 & lr-AdamW/pg1‚9.629115037250902e-08ÏDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 YKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚ 100892670 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚701462233704448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚683311949347418 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 % _timestamp‚1.7884543615710523e+09 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚691500683103744 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚11899 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚691500701106688 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _runtime‚9477.042850021 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 224552457 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚701462908069888 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10454345131008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚976 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚701462233704448 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9962206963200 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚701462908069888 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚765594483164672 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚765594295600640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 172925000 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 224552457 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 397477455 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025c²ë²EÕ*ëO UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚457 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 224572495 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚ 172924998 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 397497495 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚430 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚56902005 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚755139974409728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚701462233704448 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 9962206963200 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚701462908069888 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚765594483164672 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚20038 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚765594295600640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 163623936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 172925000 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 224552457 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 397477455 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 187564032 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚2605 YKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚ 100892670 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚701462233704448 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚683311949347418 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 8838745088 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 % _timestamp‚1.7884543615710523e+09 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚691500683103744 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚11899 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚691500701106688 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 _runtime‚9477.042850021 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚ 224552457 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚701462908069888 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10454345131008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 674365440 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚976 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 172939016 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚56906119 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 224570911 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚673441319852032 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚691557054145536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 157807420 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 trainer/global_step‚11899 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚701520080549888 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9963026404352 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9962370028544 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 397529991 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚765656990317056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚691557054145536 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 172939014 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚56906582 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10455199461888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚673441302812672 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 397529991 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 224590975 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 157806955 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 100900838 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 9477.72768008 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 165707776 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 trainer/global_step‚11899 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚701520080549888 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 9963026404352 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 9962370028544 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 397529991 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚765656990317056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚691557054145536 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 172939014 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚56906582 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 189647872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10455199461888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚673441302812672 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 397529991 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 224590975 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 157806955 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 100900838 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚ 9477.72768008 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 172939016 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚56906119 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 224570911 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚673441319852032 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚691557054145536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 157807420 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324ÑEC‰2Ù*- _wandbruntime‚9477  _runtime‚9477ħý~ZÚ*Ô 2loss/SRqqt2/assignment_loss‚1.2932978148261707 2loss/SRqqt4/assignment_loss‚1.4164399923580997 0loss/FBt2/assignment_loss‚0.3068868491469142 /loss/FBt2/detection_loss‚2.3782610850674764 &loss/total_loss‚45.661910657007496 _runtime‚9477.729111101 0loss/FRt3/assignment_loss‚1.9970668663581213 1loss/FBt1/assignment_loss‚0.14037906549234386 /loss/FRt1/detection_loss‚3.1881798749620263 1loss/SRqqt2/detection_loss‚1.7410397552781636 /loss/FBt3/detection_loss‚0.7005591472253612 trainer/global_step‚11899 2loss/SRqqt3/assignment_loss‚1.0051513638060827 0loss/FBt4/assignment_loss‚0.7525233731139451 /loss/FRt4/detection_loss‚2.0213006847238137 0loss/SRqqt1/detection_loss‚2.891530459125837 1loss/FBt3/assignment_loss‚0.18308731048616297 .loss/FRt2/detection_loss‚2.396065637895039 .loss/FBt1/detection_loss‚2.894366945539202 .loss/FBt4/detection_loss‚3.074967086315155 % _timestamp‚1.7884543622575657e+09 /loss/FRt1/assignment_loss‚4.139416683803905 1loss/SRqqt1/assignment_loss‚2.125084155549606 1loss/SRqqt3/detection_loss‚1.2443580473844822 epoch‚9 _step‚978 0loss/FRt2/assignment_loss‚3.7045060668672836 -loss/FRt4/assignment_loss‚2.6678699325707 /loss/FRt3/detection_loss‚1.7067762884000937 1loss/SRqqt4/detection_loss‚1.6927961707115173Õ°ø¦_Û*Ù 0loss/FRt3/assignment_loss‚1.9970668663581213 1loss/FBt1/assignment_loss‚0.14037906549234386 /loss/FRt1/detection_loss‚3.1881798749620263 1loss/SRqqt2/detection_loss‚1.7410397552781636 /loss/FBt3/detection_loss‚0.7005591472253612 trainer/global_step‚11899 2loss/SRqqt3/assignment_loss‚1.0051513638060827 0loss/FBt4/assignment_loss‚0.7525233731139451 /loss/FRt4/detection_loss‚2.0213006847238137 0loss/SRqqt1/detection_loss‚2.891530459125837 1loss/FBt3/assignment_loss‚0.18308731048616297 .loss/FRt2/detection_loss‚2.396065637895039 .loss/FBt1/detection_loss‚2.894366945539202 .loss/FBt4/detection_loss‚3.074967086315155 % _timestamp‚1.7884543622575657e+09 /loss/FRt1/assignment_loss‚4.139416683803905 1loss/SRqqt1/assignment_loss‚2.125084155549606 1loss/SRqqt3/detection_loss‚1.2443580473844822 epoch‚9 _step‚978 0loss/FRt2/assignment_loss‚3.7045060668672836 -loss/FRt4/assignment_loss‚2.6678699325707 /loss/FRt3/detection_loss‚1.7067762884000937 1loss/SRqqt4/detection_loss‚1.6927961707115173 2loss/SRqqt2/assignment_loss‚1.2932978148261707 2loss/SRqqt4/assignment_loss‚1.4164399923580997 0loss/FBt2/assignment_loss‚0.3068868491469142 /loss/FBt2/detection_loss‚2.3782610850674764 &loss/total_loss‚45.661910657007496 _runtime‚9477.729111101Ò¾”“Ü*‚(:ˆ ÜËæÔîíšý gpu.0.gpu‚54.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚191.909' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.3818 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0eFz°yÝ*‚(:î ÜËæÔÌ˧ÿ# disk./.usageGB‚304.609806060791 proc.cpu.threads‚17 disk.sda.out‚ 369.59765625 disk.drbd1716.out‚0 cpu‚25.335108227022385 network.sent‚ 2.9244392e+07 disk.sda3.out‚ 369.59765625 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.rbd1.in‚0 disk.rbd1.out‚ 60.0439453125 disk.nvme0n1.in‚2.65625& disk.nvme0n1.out‚38885.61181640625 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.nvme0c0n1.in‚2.65625 disk.sda3.in‚ 0.01171875( disk.nvme0c0n1.out‚38885.61181640625# disk.dm-0.out‚38885.61181640625 disk.sda.in‚ 0.01171875 disk.drbd1716.in‚0( proc.memory.availableMB‚ 1002.3046875" proc.memory.rssMB‚ 2568.1640625* proc.memory.percent‚15.674829483032227 network.recv‚ 3.36822e+06) disk./.usagePercent‚20.44261966939064 disk.drbd2080.in‚0 disk.sda1.out‚0 disk.sda2.in‚0$ memory_percent‚93.88241767883301 disk.drbd2080.out‚0 disk.drbd2266.in‚0 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd2266.out‚0 disk.dm-0.in‚2.65625 disk.sda2.out‚0 disk.dm-1.out‚ 369.59765625"׿̑Þ*‚(:† ëËæÔ阵ý gpu.0.gpu‚45.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚171.04' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚34.208 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0k8S€ß*‚(:õ ëËæԼ̎€ disk.dm-1.out‚ 370.26171875 disk.drbd2265.out‚0) proc.memory.availableMB‚ 1002.55859375 proc.cpu.threads‚17 network.recv‚ 3.373352e+06# disk.dm-0.out‚39075.78271484375 disk.sda1.in‚0" proc.memory.rssMB‚ 2568.1640625 disk.nvme0c0n1.in‚3.109375 disk.sda3.in‚ 0.01171875 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.drbd2266.in‚0 disk.drbd1716.out‚0 network.sent‚ 2.9352066e+07 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2265.in‚0* disk./.usagePercent‚20.442622229465986( disk.nvme0c0n1.out‚39075.78271484375 disk.sda3.out‚ 370.26171875 disk.sda.in‚ 0.01171875 disk.drbd2080.in‚0 disk.sda.out‚ 370.26171875 disk.rbd1.out‚ 60.4111328125 disk.nvme0n1.in‚3.109375& disk.nvme0n1.out‚39075.78271484375$ memory_percent‚93.88086795806885 disk.sda1.out‚0 disk.rbd1.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.in‚0* proc.memory.percent‚15.674829483032227 cpu‚25.334628540065737$ disk./.usageGB‚304.6098442077637 disk.drbd2080.out‚0 disk.drbd2266.out‚0 disk.dm-0.in‚3.109375R!º9“à*‚(:ˆ úËæÔÛáý gpu.0.gpu‚52.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚190.866' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚38.1732 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0„¢ü~á*‚(:ó úËæÔ¬‰­€ disk.rbd1.in‚0 disk.drbd1716.in‚0( disk.nvme0c0n1.out‚39328.77490234375( proc.memory.availableMB‚ 1002.5390625 disk.sda.in‚ 0.01171875 disk.sda.out‚ 370.83203125 disk.nvme0n1.in‚3.578125 disk.dm-1.out‚ 370.83203125 disk.drbd2080.in‚0 disk.sda3.in‚ 0.01171875 disk.sda3.out‚ 370.83203125 disk.rbd1.out‚ 60.4111328125 disk.drbd1716.out‚0 disk.drbd2080.out‚0# memory_percent‚93.8809871673584* disk./.usagePercent‚20.442622229465986$ disk./.usageGB‚304.6098442077637& disk.nvme0n1.out‚39328.77490234375 disk.dm-1.in‚ 0.01171875 disk.drbd2265.in‚0 disk.nvme0c0n1.in‚3.578125 disk.rbd0.in‚0 network.sent‚ 2.9358838e+07 disk.dm-0.in‚3.578125 disk.sda2.in‚0 disk.sda2.out‚0 disk.drbd2266.in‚0 disk.rbd0.out‚0.078125 disk.sda1.in‚0 disk.drbd2265.out‚0 disk.drbd2266.out‚0 network.recv‚ 3.378735e+06# disk.dm-0.out‚39328.77490234375 disk.sda1.out‚0" proc.memory.rssMB‚ 2568.1640625 cpu‚25.334179622329405* proc.memory.percent‚15.674829483032227 proc.cpu.threads‚17¸Çï2â*- _wandbruntime‚9513  _runtime‚9513{§IÅã* _runtime‚9513.630279478 _step‚979 & lr-AdamW/pg1‚5.248170659970596e-08 & lr-AdamW/pg2‚5.248170659970596e<¤w-I-08 trainer/global_step‚11949 % _timestamp‚1.7884543981586785e+09¦1l°Íä*Ç & lr-AdamW/pg1‚5.248170659970596e-08 & lr-AdamW/pg2‚5.248170659970596e-08 trainer/global_step‚11949 % _timestamp‚1.7884543981586785e+09 _runtime‚9513.630279478 _step‚979Óåy­õ2å*- _wandbruntime‚9513  _runtime‚9513]‚"ð'æ*êO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚686097319737710 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚704321545818112 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚694318335046144 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 399124347 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 225498547 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 158432855 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚704321545818112 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚10003192769024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚676131622249472 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 158432400 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 399104307 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚694318353049088 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚704320871452672 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚704320871452672 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9965714527598 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 _runtime‚9513.631142637 YKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚ 101301070 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 173625800 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚686097991214826 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚10003192769024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 225478509 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚758222511292928 trainer/global_step‚11949÷…€Sõ'ç*ïO UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 % _timestamp‚1.7884543981593802e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚704320871452672 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 9965714527598 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 _runtime‚9513.631142637 YKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚ 101301070 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 9326034944 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 173625800 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚453 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚686097991214826 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 9955586048 XJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚ 656362496 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚10003192769024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 225478509 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚758222511292928 trainer/global_step‚11949 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚686097319737710 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚704321545818112 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚694318335046144 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 399124347 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚ 656362496 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚ 225498547 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚ 158432855 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚704321545818112 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚10003192769024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚908 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 9643699025 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚676131622249472 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚ 158432400 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 399104307 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚694318353049088 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚704320871452672 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚704378718298112 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 158445131 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚768782624493568 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 225517027 % _timestamp‚1.7884543990708141e+09 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 158445596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚57135895 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 399156843 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚10003355834368 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 173639814 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚465 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚758284138090496 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚676186526380032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚704378718298112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 101309238 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚768782439040000 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 399136777 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚694374688084992 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚676186509340672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚694374706087936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10498462462976 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚463 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚10004012210176 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 225496963 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 173639816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 399156843 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚686153040303234 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 173639816 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _step‚981 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚9514.542826542 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 173639814 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚758284162030592 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9967185400526 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚10003355834368 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9966530962562 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚704378043919360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10498300949504„Øoê*ùM I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚9514.542826542 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 173639814 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚758284162030592 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 9967185400526 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚10003355834368 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 9966530962562 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.Í,”Aall.freed‚704378043919360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 161513472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10498300949504 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚704378718298112 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 158445131 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚768782624493568 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 VHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚ 654437964 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 225517027 % _timestamp‚1.7884543990708141e+09 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 158445596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚57135895 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 399156843 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚10003355834368 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 173639814 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚465 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚758284138090496 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚676186526380032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚704378718298112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 101309238 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚768782439040000 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 399136777 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚694374688084992 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚676186509340672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚694374706087936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10498462462976 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚463 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚10004012210176 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 225496963 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 173639816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 185453568 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 399156843 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚686153040303234 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 173639816 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _step‚981 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚11999 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚10043522212352 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 226404561 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _runtime‚9550.181372072 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚707180183566336 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚707179509200896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚678876828777472 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 174326598 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚57360753 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚771845676065280 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10540627889152 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 400751199 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚460 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚761305048176128 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10540460070912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚688883361605118 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚18002944 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 671477116 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚844 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 trainer/global_step‚11999 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚10043522212352 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 226404561 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 _runtime‚9550.181372072 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚707180183566336 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚707179509200896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 868220928 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚678876828777472 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 10194255872 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚20040 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 863815680 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚889 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚ 174Ö¸SE×326598 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚57360753 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚771845676065280 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10540627889152 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 400751199 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚460 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚761305048176128 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚10540460070912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚688883361605118 XJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚ 654437756 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚707237356046336 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 159082914 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚678931732908032 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚697192358030336 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚678931715868672 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 400763629 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 226443079 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚761366674973696 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 171999232 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10541306543616 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚697192340027392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 226423015 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 159082440 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚57364804 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚10008020739934 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚11999 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _step‚985 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 174340614 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚697192340027392 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 400783695 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚697192358030336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚10044341640192 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 174340616 SEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚ 101717636 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚57365276 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _runtime‚ 9550.86244069 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064¬Ë'ú*úM D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 400763629 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚761366698913792 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7884544353904002e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 226423015 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 10194255872 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚ 174340616 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚697192340027392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 226423015 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 159082440 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚57364804 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚414 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚10008020739934 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚11999 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 _step‚985 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 174340614 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 10194255872 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚414 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚697192340027392 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 400783695 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚697192358030336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚18002944 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚10044341640192 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 174340616 SEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚ 101717636 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚57365276 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 _runtime‚ 9550.86244069 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 101717638 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚771908177456640 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10541478542848 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚707237356046336 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 159082914 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚678931732908032 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚697192358030336 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚678931715868672 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 400763629 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 226443079 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚761366674973696 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 171999232 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10541306543616 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚889 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚691670074949642 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚10084508018176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚57588662 _runtime‚9586.471281425 UGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚ 102117868 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚681622018266112 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚12049 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 175027400 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 227330613 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10583595575296 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚691669403472526 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚710038821314560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚10085164380672 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚764387561119232 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 402358011 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚710038146949120 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚20040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚24401 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚844 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚23940096 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚691670074949642 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚18002944 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚10084508018176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚57588662 _runtime‚9586.471281425 UGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚ 102117868 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚681622018266112 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 861853521 trainer/global_step‚12049 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚2605 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚ 175027400 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚844 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚430 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚ 227330613 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚10583595575296 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚691669403472526 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚24128 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚430 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚414 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 9148669952 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 868220928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚24401 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚710038821314560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 674365440 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚10085164380672 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 863815680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 10194255872 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚414 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚414 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚24128 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 10194255872 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 868220928 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚30 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚764387561119232 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 402358011 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚710038146949120 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚681676939436032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10584460367360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 227369131 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚12049 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 175041416 SEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚ 102126036 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 227349067 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚9587.164454668 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚57593176 $ _timestamp‚1.788454471692354e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚700009991969792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚57593631 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚10048856079342 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚710095319415808 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 159719212 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚691725124038050 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚10085327446016 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 175041414 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 159719669 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 102126038 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 402410547 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚691725795515374 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 227369131 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚775033696164352 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚700010009972736 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 227349067 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚710095993794560 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚10085983821824 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 402410547 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚700009991969792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10584296756736 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚70001000997273632LŒ+úM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚24401 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚700010009972736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚764449235796992 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚764449211856896 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 8838745088 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚ 175041414 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚775033508613632 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚430 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 674378752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 863815680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚908 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚681676939436032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 868220928 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚10584460367360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 863815680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚ 227369131 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 9326034944 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 9955586048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚12049 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚20064 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚ 175041416 SEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚ 102126036 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 10194255872 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 9326034944 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 9643699025 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 674378752 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚ 227349067 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚844 _runtime‚9587.164454668 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚20064 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 339635712 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 671477324 B8DeviceStatsMonitor.on_trainÎÏ´_batch_end/active.all.current‚20066 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚57593176 $ _timestamp‚1.788454471692354e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚700009991969792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚57593631 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚414 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 9955586048 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 9148669952 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚844 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚10048856079342 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 861853521 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚430 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚710095319415808 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚ 159719212 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚23940096 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚691725124038050 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚18002944 VHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚ 656375808 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚10085327446016 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚ 175041414 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚ 159719669 WIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚ 102126038 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 402410547 SEDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚ 656375808 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚691725795515374 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚ 227369131 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚2605 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚775033696164352 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚700010009972736 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚ 227349067 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚455 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 187550720 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚844 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚710095993794560 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚10085983821824 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 402410547 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚24128 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚700009991969792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚30 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚10584296756736ÝJ#ó®2+- _wandbruntime‚9587  _runtime‚9587–Ñ˼[Ž+Õ /loss/FRt2/detection_loss‚2.2558716400103136 .loss/FRt3/detection_loss‚1.529025433374749 2loss/SRqqt1/assignment_loss‚2.1144635388928075 1loss/FBt4/assignment_loss‚0.36273746204315105 /loss/FRt1/detection_loss‚2.8515466541954964 1loss/SRqqt1/detection_loss‚2.8535334179478307 /loss/FBt4/detection_loss‚3.3080278396606446 $ _timestamp‚1.788454471694044e+09 _step‚990 0loss/FRt2/assignment_loss‚3.7387434153845818 0loss/FRt4/assignment_loss‚2.3165668538122466 2loss/SRqqt2/assignment_loss‚1.6785442296415567 .loss/FBt1/detection_loss‚3.578724645078182 2loss/SRqqt3/assignment_loss‚1.0920182299544854 2loss/SRqqt4/assignment_loss‚1.4708832755418761 %loss/total_loss‚44.12688117251605 0loss/FRt3/assignment_loss‚1.3282729483419848 1loss/FBt1/assignment_loss‚0.28340866183030045 0loss/FBt3/assignment_loss‚0.2843104881712974 1loss/SRqqt4/detection_loss‚1.4328100616378443 /loss/FRt4/detection_loss‚1.8583147591262152 0loss/SRqqt2/detection_loss‚1.951870191693306 /loss/FRt1/assignment_loss‚3.851417292248119 /loss/FBt3/detection_loss‚0.5601685338446664 epoch‚9 _runtime‚ 9587.16553668 0loss/FBt2/assignment_loss‚0.6060784711287391 1loss/SRqqt3/detection_loss‚1.0094054467194984 .loss/FBt2/detection_loss‚1.810137682236158 trainer/global_step‚12049ŒIlR`+Ú _runtime‚ 9587.16553668 0loss/FBt2/assignment_loss‚0.6060784711287391 1loss/SRqqt3/detection_loss‚1.0094054467194984 .loss/FBt2/detection_loss‚1.810137682236158 trainer/global_step‚12049 /loss/FRt2/detection_loss‚2.2558716400103136 .loss/FRt3/detection_loss‚1.529025433374749 2loss/SRqqt1/assignment_loss‚2.1144635388928075 1loss/FBt4/assignment_loss‚0.36273746204315105 /loss/FRt1/detection_loss‚2.8515466541954964 1loss/SRqqt1/detection_loss‚2.8535334179478307 /loss/FBt4/detection_loss‚3.3080278396606446 $ _timestamp‚1.788454471694044e+09 _step‚990 0loss/FRt2/assignment_loss‚3.7387434153845818 0loss/FRt4/assignment_loss‚2.3165668538122466 2loss/SRqqt2/assignment_loss‚1.6785442296415567 .loss/FBt1/detection_loss‚3.578724645078182 2loss/SRqqt3/assignment_loss‚1.0920182299544854 2loss/SRqqt4/assignment_loss‚1.4708832755418761 %loss/total_loss‚44.12688117251605 0loss/FRt3/assignment_loss‚1.3282729483419848 1loss/FBt1/assignment_loss‚0.28340866183030045 0loss/FBt3/assignment_loss‚0.2843104881712974 1loss/SRqqt4/detection_loss‚1.4328100616378443 /loss/FRt4/detection_loss‚1.8583147591262152 0loss/SRqqt2/detection_loss‚1.951870191693306 /loss/FRt1/assignment_loss‚3.851417292248119 /loss/FBt3/detection_loss‚0.5601685338446664 epoch‚9Þ ‹y'ž+‚(:“ ÔÌæÔŃ‚ý gpu.0.gpu‚55.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚38.0 gpu.0.powerWatts‚125.591' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.118199999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‹ñ¬u‘+‚(:ê ÔÌæÔºó§ÿ disk.sda2.in‚0% disk.nvme0c0n1.out‚40722.76953125& proc.memory.availableMB‚ 1001.53125# disk./.usageGB‚306.611385345459 disk.drbd1716.in‚0 disk.drbd2080.in‚0 proc.cpu.threads‚17 network.sent‚ 2.959944e+07) disk./.usagePercent‚20.57694733462159 disk.sda1.out‚0 disk.sda2.out‚0# disk.nvme0n1.out‚40722.76953125 disk.drbd2265.in‚0 disk.drbd2265.out‚0 disk.sda3.in‚ 0.01171875 disk.rbd1.out‚ 60.880859375 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125 disk.dm-0.in‚ 6.06640625 disk.sda.out‚ 374.80078125$ memory_percent‚93.88713836669922" proc.memory.rssMB‚ 2568.1640625 cpu‚25.33145133342092 disk.drbd2080.out‚0 disk.sda3.out‚ 374.80078125 network.recv‚ 3.407805e+06 disk.nvme0n1.in‚ 6.06640625 disk.nvme0c0n1.in‚ 6.06640625* proc.memory.percent‚15.674829483032227 disk.sda1.in‚0 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 374.80078125 disk.dm-0.out‚40722.76953125 disk.rbd1.in‚0 disk.drbd1716.out‚0 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.sda.in‚ 0.01171875Ù2‰tž’+‚(:“ ãÌæÔ¬óü gpu.0.gpu‚56.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚37.0 gpu.0.powerWatts‚125.156' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚25.031200000000002 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0©ý…“+‚(:ú ãÌæÔ©§ûþ disk.nvme0c0n1.in‚ 7.16796875 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.drbd1716.out‚0* proc.memory.percent‚15.674829483032227 proc.cpu.threads‚17 network.recv‚ 3.413193e+06 disk.drbd2266.in‚0 disk.drbd2266.out‚0 disk.sda.out‚ 375.51171875 disk.nvme0n1.in‚ 7.16796875 disk.drbd1716.in‚0 disk.dm-0.in‚ 7.16796875) proc.memory.availableMB‚ 1002.02734375( disk.nvme0c0n1.out‚40834.98876953125 disk.sda3.out‚ 375.51171875 disk.sda1.in‚0 disk.sda1.out‚0& disk.nvme0n1.out‚40834.98876953125 disk.drbd2265.in‚0$ memory_percent‚93.88411045074463" proc.memory.rssMB‚ 2568.1640625 network.sent‚ 2.9707117e+07* disk./.usagePercent‚20.576930182116797$ disk./.usageGB‚306.6111297607422 disk.sda3.in‚ 0.01171875 disk.sda.in‚ 0.01171875 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0# disk.dm-0.out‚40834.98876953125 disk.dm-1.in‚ 0.01171875 disk.dm-1.out‚ 375.51171875 disk.drbd2080.in‚0 cpu‚25.331015519063413 disk.rbd1.out‚ 61.162109375 disk.drbd2265.out‚0 disk.drbd2080.out‚0<…àã“”+‚(:ˆ òÌæÔ·‘œý gpu.0.gpu‚24.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚187.833' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚37.5666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÞºŠ2~•+‚(:ó òÌæÔû§½ÿ disk.sda1.in‚0 disk.drbd2080.in‚0 disk.sda.in‚ 0.01171875' proc.memory.availableMB‚ 927.0546875 disk.sda2.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0& disk.nvme0n1.out‚40931.94189453125 disk.dm-1.out‚ 376.18359375 network.sent‚ 2.971391e+07 disk.drbd2266.in‚0 disk.rbd0.in‚0 disk.dm-1.in‚ 0.01171875 disk.drbd1716.out‚0$ memory_percent‚94.34170722961426 proc.cpu.threads‚17$ disk./.usageGB‚306.6111869812012 disk.drbd2080.out‚0 disk.nvme0c0n1.in‚ 8.01171875( disk.nvme0c0n1.out‚40931.94189453125 disk.sda3.in‚ 0.01171875 disk.sda.out‚ 376.18359375 disk.rbd1.out‚ 61.341796875 disk.nvme0n1.in‚ 8.01171875) disk./.usagePercent‚20.57693402222981 disk.drbd2266.out‚0 disk.rbd0.out‚0.078125) proc.memory.percent‚16.13025665283203 cpu‚25.338133799254933 disk.drbd2265.out‚0 disk.dm-0.in‚ 8.01171875 disk.sda1.out‚0 proc.memory.rssMB‚ 2642.78125 network.recv‚ 3.418482e+06 disk.drbd2265.in‚0 disk.sda3.out‚ 376.18359375# disk.dm-0.out‚40931.94189453125 disk.drbd1716.in‚0–+‚(:“ ÍæÔºîÁÿ gpu.0.gpu‚44.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚36.0 gpu.0.powerWatts‚194.407' gpu.0.enforcedPowerLimitWatts‚500.0) gpu.0.powerPercent‚38.881400000000006 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ю‘fz—+‚(:ï ÍæÔŸø‘„ network.recv‚ 3.421709e+06 disk.dm-1.out‚ 377.07421875 disk.drbd1716.out‚0 disk.drbd2265.out‚0 disk.dm-0.in‚ 8.99609375' proc.memory.availableMB‚ 927.0234375 proc.cpu.threads‚17 network.sent‚ 2.9721384e+07$ disk.nvme0n1.out‚41009.861328125 disk.drbd2080.in‚0& disk.nvme0c0n1.out‚41009.861328125 disk.drbd2266.in‚0 disk.nvme0n1.in‚ 8.99609375 disk.drbd1716.in‚0 disk.sda3.out‚ 377.07421875) proc.memory.percent‚16.13025665283203* disk./.usagePercent‚20.576930182116797$ disk./.usageGB‚306.6111297607422 disk.sda1.in‚0 disk.sda2.out‚0 disk.rbd1.in‚0 disk.rbd1.out‚ 61.490234375 disk.drbd2265.in‚0 disk.drbd2080.out‚0 disk.sda1.out‚0 disk.sda2.in‚0 disk.sda3.in‚ 0.01171875 disk.rbd0.out‚0.078125! disk.dm-0.out‚41009.861328125$ memory_percent‚94.34189796447754 disk.sda.in‚ 0.01171875 disk.dm-1.in‚ 0.01171875 disk.nvme0c0n1.in‚ 8.99609375 disk.rbd0.in‚0 cpu‚25.345775827335725 disk.sda.out‚ 377.07421875 proc.memory.rssMB‚ 2642.78125 disk.drbd2266.out‚0ù¿K“˜+‚(:ˆ ÍæÔ¾ÕŽý gpu.0.gpu‚28.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚12.504959106445312* gpu.0.memoryAllocatedBytes‚ 10741678080 gpu.0.temp‚35.0 gpu.0.powerWatts‚122.964' gpu.0.enforcedPowerLimitWatts‚500.0 gpu.0.powerPercent‚24.5928 gpu.0.smClock‚1410 gpu.0.memoryClock‚1593! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Üí/x™+‚(:í ÍæÔ§¢Ñÿ disk.sda3.in‚ 0.01171875 disk.drbd2266.in‚0 disk.sda.in‚ 0.01171875 cpu‚25.352954166049944 disk.sda3.out‚ 377.52734375 disk.sda1.out‚0 disk.rbd1.out‚ 61.755859375 disk.drbd2265.in‚0 proc.cpu.threads‚17 disk.nvme0c0n1.in‚ 9.6640625& disk.nvme0c0n1.out‚41107.095703125 disk.drbd2266.out‚0 disk.rbd0.in‚0 disk.rbd0.out‚0.078125 disk.sda.out‚ 377.52734375) proc.memory.percent‚16.13025665283203 disk.dm-1.out‚ 377.52734375 disk.drbd1716.out‚0 disk.drbd2080.in‚0 disk.dm-0.in‚ 9.6640625$ memory_percent‚94.34125423431396 disk.sda2.in‚0 disk.sda2.out‚0 disk.nvme0n1.in‚ 9.6640625$ disk.nvme0n1.out‚41107.095703125 disk.dm-1.in‚ 0.01171875 disk.drbd2080.out‚0 proc.memory.rssMB‚ 2642.78125 disk.drbd2265.out‚0 network.sent‚ 2.9724674e+07 network.recv‚ 3.425893e+06* disk./.usagePercent‚20.576930182116797$ disk./.usageGB‚306.6111297607422 disk.drbd1716.in‚0 disk.sda1.in‚0 disk.rbd1.in‚0! disk.dm-0.out‚41107.095703125( proc.memory.availableMB‚ 927.12890625›ôl2š+- _wandbruntime‚9664  _runtime‚9664)1+æY*›+ÓT RDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚ 176131548 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚768771152305152 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚695701345695186 TEDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.peak‚ 9643699025 ]JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚10136182841856 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚714204967776768 VHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚ 176131548 OIDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚714204967776768 WIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚ 160766771 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚695701853648938 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 404597957 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 dQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚10099354775594 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510833152 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚57995317 jVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.allocated‚768771176245248 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚685602481833984 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚ 176131550 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 228466407 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 QIDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.allocated‚414 ZLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚ 102771452 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 LCDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak‚2605 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15977 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚57995164 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚ 228466407 aNDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚10136675672064 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚ 176131550 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚155 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 135089664 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 bNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚704068802937856 iVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚10636903090688 BDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 228450430 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0 YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928ð7uµóœ+ØT YKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.small_pool.peak‚ 868220928 RDDeviceStatsMonitor.on_validation_batch_start/active.large_pool.freed‚ 176131548 \ODeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.current‚18002944 fRDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.freed‚768771152305152 OGDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.current‚414 ZFDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.freed‚695701345695186 TEDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.peak‚ 9643699025 ]JDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.freed‚10136182841856 ZFDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.all.freed‚714204967776768 VHDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.freed‚ 176131548 OIDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.freed‚0 ZLDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.peak‚ 861853521 WCDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.freed‚714204967776768 WIDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.allocated‚ 160766771 ^JDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.allocated‚695701853648938 ]NDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.current‚ 9326034944 OADeviceStatsMonitor.on_validation_batch_start/active.all.allocated‚ 404597957 ZLDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.small_pool.peak‚ 863815680 NHDeviceStatsMonitor.on_validation_batch_start/oversize_segments.allocated‚0 VHDeviceStatsMonitor.on_validation_batch_start/requested_bytes.all.current‚ 507953752 dQDeviceStatsMonitor.on_validation_batch_start/requested_bytes.small_pool.allocated‚10099354775594 KEDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.freed‚0 TDDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.peak‚ 10194255872 _PDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.allocated‚ 9326034944 SEDeviceStatsMonitor.on_validation_batch_start/active_bytes.all.current‚ 510833152 E?DeviceStatsMonitor.on_validation_batch_start/num_oom_rejections‚0 OGDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.current‚430 YLDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.current‚18002944 PGDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.peak‚2605 ]PDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.allocated‚57995317 jVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.allocated‚768771176245248 aMDeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.freed‚685602481833984 RLDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_validation_batch_start/segment.all.current‚844 LDDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.peak‚908 VHDeviceStatsMonitor.on_validation_batch_start/active.large_pool.allocated‚ 176131550 J@DeviceStatsMonitor.on_validation_batch_start/allocation.all.peak‚24401 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.allocated‚ 228466407 JDDeviceStatsMonitor.on_validation_batch_start/oversize_segments.freed‚0 YIDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.all.allocated‚ 10194255872 QIDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.allocated‚414 ZLDeviceStatsMonitor.on_validation_batch_start/inactive_split.large_pool.freed‚ 102771452 SKDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.peak‚889 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.peak‚ 268325376 LCDeviceStatsMonitor.on_validation_batch_start/active.large_pool.peak‚2605 TJDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.current‚15977 YLDeviceStatsMonitor.on_validation_batch_start/inactive_split.small_pool.freed‚57995164 D>DeviceStatsMonitor.on_validation_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_validation_batch_start/num_sync_all_streams‚2 PJDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.current‚2 _QDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.peak‚ 163111424 LFDeviceStatsMonitor.on_validation_batch_start/oversize_segments.current‚0 \ODeviceStatsMonitor.on_validation_batch_start/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_validation_batch_start/segment.all.peak‚844 ZKDeviceStatsMonitor.on_validation_batch_start/reserved_bytes.large_pool.peak‚ 9326034944 VHDeviceStatsMonitor.on_validation_batch_start/active.small_pool.allocated‚ 228466407 aNDeviceStatsMonitor.on_validation_batch_start/active_bytes.small_pool.allocated‚10136675672064 [LDeviceStatsMonitor.on_validation_batch_start/allocated_bytes.large_pool.peak‚ 9148669952 ZLDeviceStatsMonitor.on_validation_batch_start/allocation.large_pool.allocated‚ 176131550 QIDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.allocated‚430 OGDeviceStatsMonitor.on_validation_batch_start/inactive_split.all.current‚155 [MDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.all.current‚ 135089664 aTDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.large_pool.current‚23940096 LFDeviceStatsMonitor.on_validation_batch_start/oversize_allocations.peak‚0 bNDeviceStatsMonitor.on_validation_batch_start/active_bytes.large_pool.allocated‚704068802937856 iVDeviceStatsMonitor.on_validation_batch_start/inactive_split_bytes.small_pool.allocated‚BØk10636903090688 BDeviceStatsMonitor.on_validation_batch_start/segment.all.freed‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.large_pool.freed‚0 VHDeviceStatsMonitor.on_validation_batch_start/allocation.small_pool.freed‚ 228450430 ICDeviceStatsMonitor.on_validation_batch_start/oversize_segments.peak‚0 KEDeviceStatsMonitor.on_validation_batch_start/segment.small_pool.freed‚0ß°Úþd2+- _wandbruntime‚9665  _runtime‚9665ð=R®c)ž+ÝR UGDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.peak‚ 863815680 G=DeviceStatsMonitor.on_validation_batch_end/active.all.current‚15979 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.peak‚ 163111424 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.freed‚0 ICDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.peak‚414 XDDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.freed‚714242162932224 ]IDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.freed‚779446185106432 @9DeviceStatsMonitor.on_validation_batch_end/max_split_size‚-1 _LDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.allocated‚10137040798720 D:DeviceStatsMonitor.on_validation_batch_end/active.all.peak‚24401 \HDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.freed‚704105614963712 RHDeviceStatsMonitor.on_validation_batch_end/allocation.small_pool.current‚15977 YKDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.current‚ 135089664 dPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.large_pool.freed‚768809028038656 \HDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.allocated‚714242673765376 PBDeviceStatsMonitor.on_validation_batch_end/active.large_pool.freed‚ 176142354 ZMDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.current‚18002944 XJDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.allocated‚ 176142356 C=DeviceStatsMonitor.on_validation_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_validation_batch_end/oversize_allocations.freed‚0 LFDeviceStatsMonitor.on_validation_batch_end/oversize_segments.allocated‚0 JDDeviceStatsMonitor.on_validation_batch_end/oversize_segments.current‚0 MEDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.current‚155 \NDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.allocated‚ 102777958 E?DeviceStatsMonitor.on_validation_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_validation_batch_end/oversize_segments.freed‚0 YJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.peak‚ 8838745088 PJDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_validation_batch_end/segment.all.allocated‚844 MEDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.current‚414 RLDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.current‚2 XJDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.peak‚ 861853521 XJDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.freed‚ 102777956 C;DeviceStatsMonitor.on_validation_batch_end/segment.all.peak‚844 ICDeviceStatsMonitor.on_validation_batch_end/segment.small_pool.freed‚0 YJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.peak‚ 9148669952 TLDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.current‚153 TFDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.freed‚ 176142354 ]ODeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.peak‚ 268325376 GADeviceStatsMonitor.on_validation_batch_end/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.allocated‚695737511017834 [LDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.current‚ 9326034944 XIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.large_pool.peak‚ 9326034944 trainer/global_step‚12089 TFDeviceStatsMonitor.on_validation_batch_end/active.large_pool.allocated‚ 176142356 RCDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.all.peak‚ 9955586048 UGDeviceStatsMonitor.on_validation_batch_end/inactive_split.all.allocated‚ 160777710 ^KDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.freed‚10099227470610 UEDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.current‚ 10194255872 % _timestamp‚1.7884545499518526e+09 cPDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.freed‚10637157067776 RCDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.peak‚ 9643699025 [MDeviceStatsMonitor.on_validation_batch_end/requested_bytes.small_pool.current‚ 490914392 MEDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.current‚430 _runtime‚9665.423844624 XJDeviceStatsMonitor.on_validation_batch_end/active_bytes.small_pool.current‚ 492830208 UADeviceStatsMonitor.on_validation_batch_end/active_bytes.all.freed‚714242162932224 PIDeviceStatsMonitor.on_validation_batch_end/inactive_split.large_pool.peak‚30 VHDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.all.peak‚ 339635712 F>DeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚704105614963712 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 111149568 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 WGDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.allocated‚ 10194255872 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 \NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.peak‚ 863815680 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚57999752 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 gTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚10637268217344 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 ^KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚10136547968512 cODeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.allocated‚685637792632832 ZMDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 VGDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak‚ 9148669952 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 PBDeviceStatsMonitor.on_validation_batch_end/active.small_pool.freed‚ 228457732 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚714242673765376 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 BDeviceStatsMonitor.on_validation_batch_end/segment.all.current‚844 OGDeviceStatsMonitor.on_validation_batch_end/segment.large_pool.allocated‚430 RBDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.peak‚ 10194255872 WIDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.peak‚ 868220928 _KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.large_pool.freed‚704105614963712 [MDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.current‚ 492830208 H>DeviceStatsMonitor.on_validation_batch_end/allocation.all.peak‚24401 WJDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.current‚18002944 `RDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.current‚ 111149568 93DeviceStatsMonitor.on_validation_batch_end/num_ooms‚0 TFDeviceStatsMonitor.on_validation_batch_end/requested_bytes.all.current‚ 507953752 WGDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.allocated‚ 10194255872 ICDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.all.freed‚0 \NDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.allocated‚ 868220928 XJDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.peak‚ 863815680 [NDeviceStatsMonitor.on_validation_batch_end/inactive_split.small_pool.allocated‚57999752 NHDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.current‚2 gTDeviceStatsMonitor.on_validation_batch_end/inactive_split_bytes.small_pool.allocated‚10637268217344 JDDeviceStatsMonitor.on_validation_batch_end/active.large_pool.current‚2 ^KDeviceStatsMonitor.on_validation_batch_end/allocated_bytes.small_pool.freed‚10136547968512 cODeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.allocated‚685637792632832 ZMDeviceStatsMonitor.on_validation_batch_end/requested_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_validation_batch_end/reserved_bytes.small_pool.current‚ 868220928 VGDeviceStatsMonitor.on_validation_batch_end/active_bytes.large_pool.peak‚ 9148669952 NDDeviceStatsMonitor.on_validation_batch_end/active.small_pool.current‚15977 PBDeviceStatsMonitor.on_validation_batch_end/active.small_pool.freed‚ 228457732 YEDeviceStatsMonitor.on_validation_batch_end/active_bytes.all.allocated‚714242673765376 O@DeviceStatsMonitor.on_validation_batch_end/active_bytes.all.peak‚ 9955586048 NEDeviceStatsMonitor.on_validation_batch_end/allocation.large_pool.peak‚2605 B&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 +jet/accuracy_3_of_3‚0.02130918949842453 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3263888955116272 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5720138549804688 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9859274625778198 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 ;"Purity/0FRt*SRqqt3FBt/event_purity‚0.048245616257190704 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5569705367088318 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5840963125228882 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.021141216158866882 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.28074711561203003 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5723429322242737 (particle/accuracy‚0.8514746427536011 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 )loss/val_total_loss‚44.86519824181869 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.17721357941627502 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.14942529797554016 :"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.47068968415260315 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.2887204885482788 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 7 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.1944444477558136 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.17308971285820007 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9744047522544861 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6161522269248962 ,"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.625 +jet/accuracy_2_of_6‚0.37820515036582947 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.46365344524383545 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5415242910385132 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 jet/accuracy_6_of_6‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5555345416069031 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 * Purity/4FRt*SRqqt*FBt/FRt_purity‚0.125 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9968725442886353 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 * Purity/1FRt0SRqqt3FBt/FRt_purity‚0.625 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0674603208899498 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.27130427956581116 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.1433093547821045 ,jet/accuracy_4_of_5‚0.012160778045654297 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.5023109912872314 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 particle/accuracy_1_of_7‚1 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.9122807383537292 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5119688510894775 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.34880948066711426 *jet/accuracy_2_of_4‚0.3706265389919281 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.47331154346466064 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 :"Purity/1FRt2SRqqt0FBt/event_purity‚0.00266400258988142 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1398998647928238 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.18985514342784882 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.027512775734066963 ,"Purity/1FRt4SRqqt0FBt/SRqqt_purityØ㘑ô‚0.375 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3810657262802124 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08287179470062256 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13661716878414154 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.44333332777023315 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5357142686843872 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5706679224967957 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5724817514419556 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4718323051929474 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 5 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.96648108959198 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.10547656565904617 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.548569917678833 ;"Purity/0FRt2SRqqt1FBt/event_purity‚0.027987422421574593 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5555555820465088 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 8"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.598758339881897 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.2158690243959427 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15053591132164001 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9255319237709045 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9913515448570251 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5728986859321594 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.6160714030265808 8 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.30000001192092896 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.13101927936077118 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5888888835906982 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 8"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.578194797039032 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3214285671710968 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 particle/accuracy_3_of_7‚1 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.4259259104728699 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.19051024317741394 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 jet/accuracy_1_of_6‚1 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 epoch‚9 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6806453466415405 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.37626487016677856 7 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.1915300190448761 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.1008041724562645 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 jet/accuracy_4_of_6‚0 ) Purity/4FRt1SRqqt0FBt/FRt_purity‚0.25 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.08006218820810318 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.014567936770617962 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.17474746704101562 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4270833134651184 jet/accuracy_7_of_7‚0 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 8"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.511742115020752 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3512517213821411 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.2665116786956787 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9836550354957581 7 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.1395755410194397 jet/accuracy_5_of_7‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5492808222770691 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 *jet/accuracy_1_of_5‚0.9688428640365601 particle/accuracy_3_of_6‚1 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9957951307296753 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5741448998451233 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 *jet/accuracy_1_of_1‚0.5188506841659546 ,jet/accuracy_4_of_4‚0.002807604381814599 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.6176470518112183 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9867123961448669 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.12812255322933197 ;"Purity/1FRt2SRqqt1FBt/event_purity‚0.003448275849223137 ;"Purity/2FRt*SRqqt2FBt/event_purity‚0.013565892353653908 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.44476744532585144 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 particle/accuracy_1_of_3‚1 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.1388888955116272 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.47069716453552246 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9886267185211182 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.30970388650894165 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9479167461395264 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5607143044471741 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6437471508979797 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5676147937774658 particle/accuracy_2_of_4‚1 +jet/accuracy_3_of_6‚0.13461539149284363 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 +particle/specificity‚0.9476725459098816 particle/accuracy_2_of_6‚1 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5752274990081787 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ,"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.375 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.6081504821777344 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 (jet/accuracy_1_of_4‚0.92777419090271 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5777392387390137 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.5862764716148376 particle/accuracy_5_of_6‚1 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.44729727506637573 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.7708333134651184 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.040009524673223495 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.20000000298023224 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.980235755443573 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.14023247361183167 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 7 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.1111111119389534 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0030595569405704737 ,jet/accuracy_3_of_4‚0.050929293036460876 particle/accuracy_3_of_4‚1 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 7 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.2675926089286804 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 jet/accuracy_4_of_7‚0 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.11489463597536087 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.14173075556755066 particle/accuracy_4_of_6‚1 particle/accuracy_4_of_7‚1 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.03868537023663521 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.4197531044483185 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.6087963581085205 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.40367475152015686 8"Purity/0FRt2SRqqt0FBt/event_purity‚0.096190445125103 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 8"Purity/*FRt0SRqqt0FBt/event_purity‚0.151108980178833 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5168694257736206 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5502572655677795 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 +jet/accuracy_3_of_5‚0.08776241540908813 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2060794234275818 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5619015693664551 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9936092495918274 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.15535202622413635 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 & Purity/1FRt0SRqqt4FBt/FRt_purity‚1 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 particle/accuracy_6_of_7‚1 _step‚993 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 particle/accuracy_2_of_3‚1 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.24298270046710968 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9896175265312195 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.3513777554035187 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.25970515608787537 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.3927689790725708 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.6066666841506958 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 jet/accuracy_1_of_7‚1 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.01247055921703577 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.08541865646839142 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.26660066843032837 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 particle/accuracy_1_of_6‚1 particle/accuracy_5_of_7‚1 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 7 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.1315789520740509 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.013559668324887753 jet/accuracy_5_of_6‚0 jet/accuracy_3_of_7‚0 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.2430351823568344 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.2983870804309845 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.6114374995231628 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.09047619253396988 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.46531984210014343 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_4‚1 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9849824905395508 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5295748114585876 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09205086529254913 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.42872804403305054 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 particle/accuracy_1_of_5‚1 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.33289802074432373 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.998068630695343 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 *jet/accuracy_1_of_2‚0.7728320956230164 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.12000001221895218 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.6139830350875854 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8136881589889526 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6189219355583191 6 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.225198432803154 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4565599858760834 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.17721357941627502 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.2041894644498825 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 * Purity/4FRt*SRqqt0FBt/FRt_purity‚0.125 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9907218813896179 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 +"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.25 trainer/global_step‚12089 5 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.98872971534729 ;"Purity/2FRt0SRqqt2FBt/event_purity‚0.015350878238677979 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.035354550927877426 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.4103299677371979 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.6034747958183289 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.007248338777571917 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.10594291239976883 particle/accuracy_7_of_7‚1 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9864341616630554 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 jet/accuracy_6_of_7‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5689042210578918 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 8"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.618739664554596 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.6228070259094238 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 & Purity/1FRt1SRqqt3FBt/FRt_purity‚0 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17954924702644348 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.3124999701976776 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9533333778381348 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5775101184844971 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.3125 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5526315569877625 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 8"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.532017171382904 *jet/accuracy_2_of_5‚0.4947388172149658 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.04964539408683777 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.5008620619773865 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 +jet/accuracy_2_of_3‚0.24703650176525116 7validation_average_jet_accuracy‚0.45350801944732666 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15098799765110016 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.0555555559694767 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 particle/accuracy_1_of_1‚1 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.34375 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.2124755084514618 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.4048507809638977 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.4431818127632141 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4000000059604645 jet/accuracy_2_of_7‚0.5 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 % _timestamp‚1.7884545629111624e+09 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 particle/accuracy_5_of_5‚1 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5919520854949951 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24293671548366547 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.2534162402153015 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5582482814788818 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 particle/accuracy_4_of_4‚1 7 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.3574073612689972 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 7 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.1291554719209671 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 'particle/f_score‚0.4105576276779175 9"Purity/0FRt0SRqqt3FBt/event_purity‚0.0572916679084301 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 ) Purity/4FRt1SRqqt*FBt/FRt_purity‚0.25 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5741655230522156 7 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.1828828901052475 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.3645833432674408 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 ,particle/sensitivity‚0.33154258131980896 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17898771166801453 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9444443583488464 7 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.1498260200023651 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 8 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.34947335720062256 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 particle/accuracy_3_of_3‚1 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5593216419219971 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3263888955116272 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.1777586042881012 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9683907628059387 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9896711111068726 8 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.44097229838371277 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.1388888955116272 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.46033909916877747 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.2380952537059784 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9464285969734192 particle/accuracy_2_of_7‚1 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9881795644760132 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5368471741676331 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7424241900444031 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 jet/accuracy_5_of_5‚0 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6005747318267822 particle/accuracy_4_of_5‚1 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5911991596221924 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9914966225624084 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5706679224967957 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 particle/accuracy_6_of_6‚1 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.46019625663757324 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.7708333134651184 _runtime‚9678.385028062 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.03751574456691742 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9805707335472107 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9765334129333496 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.20000000298023224 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5641942620277405 6 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.988304078578949 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7121211886405945 & Purity/*FRt1SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9925675988197327 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.18758326768875122 8"Purity/0FRt*SRqqt*FBt/event_purity‚0.504177451133728 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.09670782089233398 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8888064026832581 particle/accuracy_1_of_2‚1 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.15909090638160706 & Purity/*FRt0SRqqt4FBt/FRt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14307044446468353 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9628472924232483 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11197067052125931 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.0625 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_5‚1 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.015456990338861942 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.02335028350353241 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9937499761581421 +"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.25 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.20982758700847626 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 7"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.53165203332901 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9809297919273376 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.24813145399093628 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.5864197611808777 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9936118125915527 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9659090638160706 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.4704919755458832 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.015362811274826527 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9872742891311646 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0>fÁù¤+–¦ 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9936092495918274 7 Purity/*FRt1SRqqt3FBt/FBt_purity‚0.8518518209457397 & Purity/2FRt2SRqqt2FBt/FBt_purity‚1 <&Purity/*FRt*SRqqt2FBt/event_proportion‚0.067336305975914 @&Purity/*FRt1SRqqt3FBt/event_proportion‚0.0006820436683483422 <&Purity/1FRt1SRqqt*FBt/event_proportion‚0.172185018658638 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.15535202622413635 >&Purity/3FRt*SRqqt*FBt/event_proportion‚0.01016865111887455 & Purity/1FRt0SRqqt4FBt/FRt_purity‚1 & Purity/2FRt*SRqqt3FBt/FBt_purity‚1 ,&Purity/4FRt4SRqqt0FBt/event_proportion‚0 & Purity/*FRt4SRqqt1FBt/FBt_purity‚1 particle/accuracy_6_of_7‚1 _step‚993 ?&Purity/2FRt3SRqqt*FBt/event_proportion‚0.003968254197388887 ,&Purity/3FRt*SRqqt3FBt/event_proportion‚0 @&Purity/3FRt1SRqqt*FBt/event_proportion‚0.0037202381063252687 particle/accuracy_2_of_3‚1 9"Purity/*FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ("Purity/4FRt2SRqqt0FBt/SRqqt_purity‚1 :"Purity/*FRt0SRqqt2FBt/event_purity‚0.24298270046710968 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9896175265312195 7 Purity/2FRt2SRqqt*FBt/FBt_purity‚0.9767441749572754 ,&Purity/4FRt4SRqqt4FBt/event_proportion‚0 & Purity/3FRt2SRqqt1FBt/FBt_purity‚1 A&Purity/1FRt*SRqqt4FBt/event_proportion‚0.00012400794366840273 =&Purity/1FRt2SRqqt*FBt/event_proportion‚0.0573536716401577 ,&Purity/1FRt4SRqqt1FBt/event_proportion‚0 & Purity/3FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.3513777554035187 ,&Purity/4FRt2SRqqt1FBt/event_proportion‚0 ("Purity/2FRt0SRqqt3FBt/event_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.25970515608787537 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.3927689790725708 A&Purity/1FRt1SRqqt3FBt/event_proportion‚0.00024801588733680546 9"Purity/3FRt1SRqqt0FBt/SRqqt_purity‚0.6066666841506958 ("Purity/2FRt3SRqqt1FBt/event_purity‚0 jet/accuracy_1_of_7‚1 ("Purity/*FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/2FRt2SRqqt2FBt/SRqqt_purity‚0 ("Purity/*FRt3SRqqt*FBt/event_purity‚0 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.01247055921703577 ,&Purity/4FRt3SRqqt1FBt/event_proportion‚0 ("Purity/*FRt2SRqqt2FBt/event_purity‚0 ("Purity/0FRt2SRqqt2FBt/event_purity‚0 ("Purity/0FRt4SRqqt0FBt/event_purity‚0 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.08541865646839142 ("Purity/3FRt*SRqqt*FBt/event_purity‚0 & Purity/0FRt3SRqqt1FBt/FBt_purity‚1 ("Purity/4FRt1SRqqt*FBt/SRqqt_purity‚1 ,&Purity/0FRt3SRqqt3FBt/event_proportion‚0 5&Purity/2FRt2SRqqt1FBt/event_proportion‚ 0.00390625 & Purity/1FRt3SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.26660066843032837 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0407366082072258 ("Purity/1FRt3SRqqt*FBt/event_purity‚0 particle/accuracy_1_of_6‚1 particle/accuracy_5_of_7‚1 & Purity/3FRt1SRqqt2FBt/FBt_purity‚1 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2310267835855484 7 Purity/1FRt3SRqqt0FBt/FRt_purity‚0.1315789520740509 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.013559668324887753 jet/accuracy_5_of_6‚0 jet/accuracy_3_of_7‚0 ("Purity/0FRt1SRqqt3FBt/event_purity‚0 ("Purity/1FRt3SRqqt2FBt/event_purity‚0 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.2430351823568344 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.2983870804309845 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.6114374995231628 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.09047619253396988 :"Purity/2FRt3SRqqt0FBt/SRqqt_purity‚0.46531984210014343 @&Purity/2FRt4SRqqt*FBt/event_proportion‚0.0003100198518950492 ("Purity/3FRt*SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_4‚1 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9849824905395508 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.5295748114585876 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09205086529254913 @&Purity/2FRt*SRqqt3FBt/event_proportion‚6.200397183420137e-05 8 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.42872804403305054 ,&Purity/2FRt4SRqqt1FBt/event_proportion‚0 @&Purity/3FRt*SRqqt2FBt/event_proportion‚0.0003100198518950492 particle/accuracy_1_of_5‚1 @&Purity/*FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.4109002947807312 ,&Purity/1FRt4SRqqt2FBt/event_proportion‚0 @&Purity/3FRt3SRqqt0FBt/event_proportion‚0.0005580357392318547 ,&Purity/4FRt1SRqqt1FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 ("Purity/3FRt1SRqqt2FBt/event_purity‚0 & Purity/1FRt4SRqqt0FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.1462053507566452 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0482390858232975 :"Purity/0FRt0SRqqt2FBt/event_purity‚0.33289802074432373 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.998068630695343 ?&Purity/3FRt*SRqqt0FBt/event_proportion‚0.007254464086145163 *jet/accuracy_1_of_2‚0.7728320956230164 8 Purity/3FRt1SRqqt0FBt/FRt_purity‚0.12000001221895218 ("Purity/*FRt4SRqqt*FBt/event_purity‚0 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.6139830350875854 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.8136881589889526 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6189219355583191 6 Purity/2FRt2SRqqt1FBt/FRt_purity‚0.225198432803154 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.4565599858760834 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.17721357941627502 7 Purity/1FRt0SRqqt3FBt/FBt_purity‚0.9583333134651184 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.2041894644498825 ,&Purity/4FRt3SRqqt4FBt/event_proportion‚0 9"Purity/3FRt2SRqqt1FBt/SRqqt_purity‚0.5714285969734192 * Purity/4FRt*SRqqt0FBt/FRt_purity‚0.125 & Purity/4FRt2SRqqt*FBt/FRt_purity‚0 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9907218813896179 @&Purity/3FRt1SRqqt1FBt/event_proportion‚0.0009920635493472219 ,&Purity/3FRt4SRqqt2FBt/event_proportion‚0 +"Purity/0FRt0SRqqt4FBt/event_purity‚0.25 +"Purity/2FRt4SRqqt*FBt/SRqqt_purity‚0.25 trainer/global_step‚12089 5 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.98872971534729 ;"Purity/2FRt0SRqqt2FBt/event_purity‚0.015350878238677979 ,&Purity/2FRt3SRqqt4FBt/event_proportion‚0 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.035354550927877426 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.4103299677371979 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.6034747958183289 ,&Purity/3FRt0SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt2FBt/FRt_purity‚0 & Purity/*FRt0SRqqt1FBt/FBt_purity‚1 @&Purity/*FRt4SRqqt*FBt/event_proportion‚0.0013020833721384406 ,&Purity/0FRt3SRqqt4FBt/event_proportion‚0 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.007248338777571917 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.10594291239976883 particle/accuracy_7_of_7‚1 7 Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9864341616630554 =&Purity/2FRt1SRqqt0FBt/event_proportion‚0.0318080373108387 @&Purity/3FRt3SRqqt*FBt/event_proportion‚0.0005580357392318547 jet/accuracy_6_of_7‚0 & Purity/1FRt0SRqqt4FBt/FBt_purity‚1 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5689042210578918 ,&Purity/0FRt2SRqqt4FBt/event_proportion‚0 ("Purity/3FRt1SRqqt1FBt/event_purity‚0 ("Purity/*FRt2SRqqt3FBt/event_purity‚0 8"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.618739664554596 ,&Purity/1FRt3SRqqt4FBt/event_proportion‚0 9"Purity/3FRt*SRqqt1FBt/SRqqt_purity‚0.6228070259094238 7 Purity/3FRt4SRqqt*FBt/FRt_purity‚0.3333333432674408 & Purity/1FRt1SRqqt3FBt/FRt_purity‚0 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.17954924702644348 @&Purity/4FRt2SRqqt*FBt/event_proportion‚6.200397183420137e-05 ,&Purity/4FRt4SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt1SRqqt1FBt/FRt_purity‚0.3124999701976776 7 Purity/*FRt2SRqqt2FBt/FBt_purity‚0.9533333778381348 ,&Purity/3FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt3FBt/event_proportion‚0 @&Purity/0FRt1SRqqt3FBt/event_proportion‚0.0004340277810115367 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5775101184844971 + Purity/1FRt3SRqqt1FBt/FRt_purity‚0.3125 7 Purity/*FRt*SRqqt3FBt/FRt_purity‚0.5526315569877625 ("Purity/*FRt3SRqqt0FBt/event_purity‚0 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 8"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.532017171382904 *jet/accuracy_2_of_5‚0.4947388172149658 ("Purity/3FRt4SRqqt*FBt/event_purity‚0 :"Purity/*FRt*SRqqt3FBt/event_purity‚0.04964539408683777 ?&Purity/0FRt3SRqqt0FBt/event_proportion‚0.006076388992369175 >&Purity/1FRt0SRqqt2FBt/event_proportion‚0.01512896828353405 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.5008620619773865 ,&Purity/2FRt1SRqqt4FBt/event_proportion‚0 ,&Purity/2FRt4SRqqt3FBt/event_proportion‚0 +jet/accuracy_2_of_3‚0.24703650176525116 7validation_average_jet_accuracy‚0.45350801944732666 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15098799765110016 9"Purity/1FRt*SRqqt3FBt/event_purity‚0.0555555559694767 7 Purity/1FRt*SRqqt3FBt/FBt_purity‚0.9444444179534912 >&Purity/2FRt*SRqqt2FBt/event_proportion‚0.00533234141767025 particle/accuracy_1_of_1‚1 7 Purity/*FRt0SRqqt4FBt/FBt_purity‚0.8999999761581421 & Purity/*FRt4SRqqt*FBt/FBt_purity‚1 ."Purity/0FRt4SRqqt*FBt/SRqqt_purity‚0.34375 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.2124755084514618 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.4048507809638977 ,&Purity/1FRt2SRqqt3FBt/event_proportion‚0 7 Purity/2FRt1SRqqt2FBt/FRt_purity‚0.4431818127632141 ("Purity/4FRt2SRqqt*FBt/event_purity‚0 @&Purity/0FRt4SRqqt*FBt/event_proportion‚0.0004960317746736109 ("Purity/1FRt3SRqqt1FBt/event_purity‚0 9"Purity/1FRt2SRqqt2FBt/SRqqt_purity‚0.4000000059604645 jet/accuracy_2_of_7‚0.5 ( Purity/2FRt0SRqqt3FBt/FRt_purity‚0.5 & Purity/3FRt0SRqqt2FBt/FBt_purity‚1 ("Purity/0FRt2SRqqt3FBt/event_purity‚0 % _timestamp‚1.7884545629111624e+09 ("Purity/2FRt3SRqqt*FBt/event_purity‚0 A&Purity/4FRt*SRqqt*FBt/event_proportion‚0.00018601190822664648 particle/accuracy_5_of_5‚1 ("Purity/3FRt3SRqqt*FBt/event_purity‚0 ("Purity/1FRt1SRqqt3FBt/event_purity‚0 =&Purity/1FRt*SRqqt1FBt/event_proportion‚0.0987723246216774 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.1308283656835556 @&Purity/1FRt0SRqqt4FBt/event_proportion‚6.200397183420137e-05 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0416666679084301 ,&Purity/4FRt*SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5919520854949951 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24293671548366547 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.2534162402153015 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5582482814788818 ,&Purity/3FRt3SRqqt1FBt/event_proportion‚0 particle/accuracy_4_of_4‚1 7 Purity/3FRt*SRqqt1FBt/FRt_purity‚0.3574073612689972 & Purity/3FRt1SRqqt1FBt/FBt_purity‚1 A&Purity/1FRt4SRqqt*FBt/event_proportion‚0.00037202381645329297 ,&Purity/2FRt0SRqqt4FBt/event_proportion‚0 7 Purity/3FRt*SRqqt0FBt/FRt_purity‚0.1291554719209671 @&Purity/3FRt0SRqqt2FBt/event_proportion‚6.200397183420137e-05 'particle/f_score‚0.4105576276779175 9"Purity/0FRt0SRqqt3FBt/event_purity‚0.0572916679084301 @&Purity/*FRt*SRqqt3FBt/event_proportion‚0.0045262896455824375 >&Purity/*FRt3SRqqt*FBt/event_proportion‚0.02021329291164875 & Purity/0FRt3SRqqt*FBt/FBt_purity‚1 ) Purity/4FRt1SRqqt*FBt/FRt_purity‚0.25 =&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0225694440305233 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5741655230522156 7 Purity/2FRt3SRqqt*FBt/FRt_purity‚0.1828828901052475 ,&Purity/2FRt4SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt*SRqqt3FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt3FBt/event_proportion‚0 ("Purity/1FRt4SRqqt*FBt/event_purity‚0 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.3645833432674408 =&Purity/1FRt2SRqqt0FBt/event_proportion‚0.0474950410425663 ,particle/sensitivity‚0.33154258131980896 ("Purity/0FRt3SRqqt1FBt/event_purity‚0 ("Purity/*FRt1SRqqt3FBt/event_purity‚0 ("Purity/4FRt*SRqqt*FBt/SRqqt_purity‚1 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.3558407723903656 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17898771166801453 7 Purity/*FRt0SRqqt3FBt/FBt_purity‚0.9444443583488464 7 Purity/*FRt3SRqqt*FBt/FRt_purity‚0.1498260200023651 6 Purity/2FRt2SRqqt1FBt/FBt_purity‚0.976190447807312 ,&Purity/2FRt3SRqqt2FBt/event_proportion‚0 & Purity/3FRt2SRqqt*FBt/FBt_purity‚1 8 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.34947335720062256 ("Purity/2FRt2SRqqt1FBt/event_purity‚0 ,&Purity/3FRt1SRqqt3FBt/event_proportion‚0 particle/accuracy_3_of_3‚1 & Purity/3FRt0SRqqt*FBt/FBt_purity‚1 ("Purity/2FRt4SRqqt0FBt/event_purity‚0 @&Purity/2FRt2SRqqt2FBt/event_proportion‚6.200397183420137e-05 @&Purity/3FRt0SRqqt0FBt/event_proportion‚0.0016121031949296594 ("Purity/3FRt2SRqqt0FBt/event_purity‚0 ("Purity/3FRt0SRqqt1FBt/event_purity‚0 5 Purity/0FRt1SRqqt3FBt/FBt_purity‚0.80952388048172 9"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.5593216419219971 @&Purity/4FRt1SRqqt0FBt/event_proportion‚6.200397183420137e-05 9"Purity/*FRt4SRqqt0FBt/SRqqt_purity‚0.3263888955116272 ("Purity/0FRt3SRqqt*FBt/event_purity‚0 * Purity/1FRt*SRqqt4FBt/FBt_purity‚0.875 7 Purity/0FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.1777586042881012 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9683907628059387 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9896711111068726 8 Purity/3FRt0SRqqt1FBt/FRt_purity‚0.44097229838371277 7 Purity/*FRt4SRqqt0FBt/FRt_purity‚0.1388888955116272 ("Purity/0FRt3SRqqt0FBt/event_purity‚0 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.46033909916877747 ,&Purity/4FRt1SRqqt3FBt/event_proportion‚0 7 Purity/3FRt2SRqqt1FBt/FRt_purity‚0.2380952537059784 7 Purity/0FRt2SRqqt2FBt/FBt_purity‚0.9464285969734192 particle/accuracy_2_of_7‚1 ( Purity/1FRt*SRqqt4FBt/FRt_purity‚0.5 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9881795644760132 ,&Purity/*FRt4SRqqt2FBt/event_proportion‚0 @&Purity/2FRt4SRqqt0FBt/event_proportion‚0.0003100198518950492 ,&Purity/3FRt3SRqqt3FBt/event_proportion‚0 ("Purity/1FRt4SRqqt0FBt/event_purity‚0 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.5368471741676331 7 Purity/2FRt*SRqqt1FBt/FBt_purity‚0.9960317611694336 9"Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0.7424241900444031 ?&Purity/3FRt0SRqqt1FBt/event_proportion‚0.001178075443021953 ,&Purity/4FRt*SRqqt4FBt/event_proportion‚0 jet/accuracy_5_of_5‚0 9"Purity/3FRt2SRqqt*FBt/SRqqt_purity‚0.6005747318267822 particle/accuracy_4_of_5‚1 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.5911991596221924 7 Purity/1FRt2SRqqt2FBt/FRt_purity‚0.2666666805744171 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9914966225624084 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5706679224967957 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ("Purity/1FRt3SRqqt0FBt/event_purity‚0 ,&Purity/4FRt2SRqqt3FBt/event_proportion‚0 & Purity/3FRt*SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.2714533805847168 =&Purity/1FRt2SRqqt1FBt/event_proportion‚0.0088665671646595 ,&Purity/1FRt2SRqqt4FBt/event_proportion‚0 & Purity/4FRt0SRqqt0FBt/FRt_purity‚0 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 =&Purity/2FRt2SRqqt*FBt/event_proportion‚0.0193452388048172 ,&Purity/4FRt3SRqqt2FBt/event_proportion‚0 ,"Purity/0FRt*SRqqt3FBt/SRqqt_purity‚0.875 ("Purity/1FRt1SRqqt4FBt/SRqqt_purity‚1 >&Purity/*FRt3SRqqt0FBt/event_proportion‚0.01804315485060215 ,&Purity/3FRt0SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt0FBt/event_proportion‚0.00012400794366840273 particle/accuracy_6_of_6‚1 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.46019625663757324 ,&Purity/2FRt3SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt*SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt*FBt/event_proportion‚0 & Purity/3FRt*SRqqt*FBt/FBt_purity‚1 9"Purity/3FRt3SRqqt0FBt/SRqqt_purity‚0.7708333134651184 _runtime‚9678.385028062 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.03751574456691742 ("Purity/3FRt4SRqqt0FBt/event_purity‚0 ,&Purity/*FRt4SRqqt3FBt/event_proportion‚0 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9805707335472107 @&Purity/0FRt4SRqqt0FBt/event_proportion‚0.0004340277810115367 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9765334129333496 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0758928582072258 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 8 Purity/2FRt4SRqqt*FBt/FRt_purity‚0.20000000298023224 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5641942620277405 6 Purity/0FRt2SRqqt*FBt/FBt_purity‚0.988304078578949 9"Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0.7121211886405945 & Purity/*FRt1SRqqt3FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9925675988197327 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.18758326768875122 8"Purity/0FRt*SRqqt*FBt/event_purity‚0.504177451133728 8 Purity/3FRt2SRqqt0FBt/FRt_purity‚0.09670782089233398 *"Purity/0FRt4SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/2FRt*SRqqt3FBt/event_purity‚0 ("Purity/4FRt1SRqqt0FBt/event_purity‚0 ,&Purity/2FRt2SRqqt4FBt/event_proportion‚0 ,&Purity/3FRt2SRqqt4FBt/event_proportion‚0 *jet/accuracy_1_of_3‚0.8888064026832581 particle/accuracy_1_of_2‚1 8 Purity/3FRt0SRqqt0FBt/FRt_purity‚0.15909090638160706 & Purity/*FRt0SRqqt4FBt/FRt_purity‚1 7 Purity/3FRt3SRqqt0FBt/FRt_purity‚0.0833333358168602 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.14307044446468353 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9628472924232483 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.11197067052125931 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 @&Purity/3FRt2SRqqt0FBt/event_proportion‚0.0024801588151603937 *"Purity/3FRt1SRqqt2FBt/SRqqt_purity‚0.5 @&Purity/3FRt*SRqqt1FBt/event_proportion‚0.0026041667442768812 ("Purity/1FRt*SRqqt4FBt/event_purity‚0 ("Purity/*FRt1SRqqt4FBt/event_purity‚0 -"Purity/1FRt0SRqqt3FBt/event_purity‚0.0625 ,&Purity/2FRt1SRqqt3FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt3FBt/event_proportion‚0 particle/accuracy_2_of_5‚1 9"Purity/1FRt3SRqqt2FBt/SRqqt_purity‚0.6666666865348816 ;"Purity/*FRt1SRqqt2FBt/event_purity‚0.015456990338861942 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.02335028350353241 ,&Purity/0FRt3SRqqt2FBt/event_proportion‚0 @&Purity/1FRt3SRqqt2FBt/event_proportion‚6.200397183420137e-05 ,&Purity/1FRt4SRqqt3FBt/event_proportion‚0 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9937499761581421 +"Purity/2FRt4SRqqt0FBt/SRqqt_purity‚0.25 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.4355778694152832 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.20982758700847626 9"Purity/2FRt3SRqqt1FBt/SRqqt_purity‚0.3888888657093048 7"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.53165203332901 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9809297919273376 4&Purity/1FRt3SRqqt0FBt/event_proportion‚ 0.0078125 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ,&Purity/3FRt3SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt4SRqqt*FBt/event_proportion‚0 ("Purity/*FRt4SRqqt0FBt/event_purity‚0 ( Purity/2FRt*SRqqt3FBt/FRt_purity‚0.5 :"Purity/0FRt*SRqqt2FBt/event_purity‚0.24813145399093628 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 @&Purity/1FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ,&Purity/1FRt4SRqqt4FBt/event_proportion‚0 9"Purity/3FRt2SRqqt0FBt/SRqqt_purity‚0.5864197611808777 ,&Purity/3FRt2SRqqt2FBt/event_proportion‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9936118125915527 ,&Purity/*FRt2SRqqt4FBt/event_proportion‚0 =&Purity/1FRt0SRqqt1FBt/event_proportion‚0.0495411716401577 7 Purity/2FRt1SRqqt2FBt/FBt_purity‚0.9659090638160706 ("Purity/2FRt4SRqqt*FBt/event_purity‚0 9"Purity/*FRt3SRqqt0FBt/SRqqt_purity‚0.4704919755458832 =&Purity/0FRt2SRqqt0FBt/event_proportion‚0.0431547611951828 ;"Purity/*FRt2SRqqt1FBt/event_purity‚0.015362811274826527 @&Purity/*FRt2SRqqt2FBt/event_proportion‚0.0020461310632526875 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9872742891311646 ,&Purity/0FRt4SRqqt4FBt/event_proportion‚0 ,&Purity/4FRt0SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_2‚1 & Purity/0FRt4SRqqt1FBt/FBt_purity‚1 7 Purity/3FRt0SRqqt2FBt/FRt_purity‚0.3333333432674408 @&Purity/*FRt0SRqqt4FBt/event_proportion‚0.0003100198518950492 >&Purity/0FRt2SRqqt1FBt/event_proportion‚0.00936259888112545 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.3129960298538208 +jet/accuracy_3_of_3‚0.02130918949842453 9"Purity/*FRt*SRqqt4FBt/event_purity‚0.1666666716337204 9"Purity/*FRt4SRqqt*FBt/SRqqt_purity‚0.3263888955116272 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5720138549804688 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9859274625778198 A&Purity/1FRt4SRqqt0FBt/event_proportion‚0.00037202381645329297 ,&Purity/3FRt1SRqqt4FBt/event_proportion‚0 ;"Purity/0FRt*SRqqt3FBt/event_purity‚0.048245616257190704 ("Purity/*FRt4SRqqt1FBt/event_purity‚0 ,&Purity/0FRt1SRqqt4FBt/event_proportion‚0 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.5569705367088318 & Purity/1FRt1SRqqt4FBt/FRt_purity‚0 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.5840963125228882 @&Purity/0FRt2SRqqt2FBt/event_proportion‚0.0009920635493472219 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ("Purity/3FRt0SRqqt*FBt/event_purity‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.1489335298538208 ;"Purity/0FRt1SRqqt2FBt/event_purity‚0.021141216158866882 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.28074711561203003 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5723429322242737 (particle/accuracy‚0.8514746427536011 ("Purity/*FRt3SRqqt1FBt/event_purity‚0 )loss/val_total_loss‚44.86519824181869 ,&Purity/*FRt3SRqqt3FBt/event_proportion‚0 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.17721357941627502 8 Purity/1FRt3SRqqt*FBt/FRt_purity‚0.14942529797554016 :"Purity/1FRt3SRqqt*FBt/SRqqt_purity‚0.47068968415260315 ("Purity/0FRt2SRqqt3FBt/SRqqt_purity‚1 @&Purity/*FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.1966765820980072 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.2887204885482788 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 ,&Purity/4FRt0SRqqt4FBt/event_proportion‚0 7 Purity/3FRt1SRqqt*FBt/FRt_purity‚0.1944444477558136 ("Purity/1FRt1SRqqt3FBt/SRqqt_purity‚1 A&Purity/*FRt*SRqqt4FBt/event_proportion‚0.00037202381645329297 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.17308971285820007 7 Purity/1FRt2SRqqt*FBt/FBt_purity‚0.9744047522544861 9"Purity/3FRt*SRqqt*FBt/SRqqt_purity‚0.6161522269248962 ,"Purity/3FRt1SRqqt1FBt/SRqqt_purity‚0.625 +jet/accuracy_2_of_6‚0.37820515036582947 *"Purity/3FRt4SRqqt0FBt/SRqqt_purity‚0.5 ("Purity/3FRt*SRqqt2FBt/event_purity‚0 :"Purity/*FRt3SRqqt*FBt/SRqqt_purity‚0.46365344524383545 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5415242910385132 ,&Purity/2FRt*SRqqt4FBt/event_proportion‚0 A&Purity/3FRt4SRqqt*FBt/event_proportion‚0.00012400794366840273 ("Purity/3FRt*SRqqt1FBt/event_purity‚0 jet/accuracy_6_of_6‚0 & Purity/1FRt4SRqqt*FBt/FRt_purity‚0 ) Purity/1FRt1SRqqt4FBt/FBt_purity‚0.75 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.5555345416069031 ("Purity/3FRt3SRqqt0FBt/event_purity‚0 * Purity/4FRt*SRqqt*FBt/FRt_purity‚0.125 & Purity/4FRt2SRqqt0FBt/FRt_purity‚0 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9968725442886353 @&Purity/*FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 @&Purity/1FRt*SRqqt3FBt/event_proportion‚0.0013640873366966844 * Purity/1FRt0SRqqt3FBt/FRt_purity‚0.625 ,&Purity/4FRt1SRqqt4FBt/event_proportion‚0 * Purity/0FRt*SRqqt4FBt/FBt_purity‚0.875 8"Purity/*FRt*SRqqt3FBt/SRqqt_purity‚0.949999988079071 ("Purity/*FRt3SRqqt2FBt/event_purity‚0 9"Purity/*FRt0SRqqt3FBt/event_purity‚0.0674603208899498 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.1695808470249176 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.27130427956581116 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.1433093547821045 ,jet/accuracy_4_of_5‚0.012160778045654297 ("Purity/1FRt*SRqqt3FBt/SRqqt_purity‚1 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.5023109912872314 A&Purity/2FRt3SRqqt1FBt/event_proportion‚0.00037202381645329297 particle/accuracy_1_of_7‚1 ("Purity/4FRt1SRqqt0FBt/SRqqt_purity‚1 >&Purity/*FRt1SRqqt2FBt/event_proportion‚0.01698908768594265 7 Purity/0FRt*SRqqt3FBt/FBt_purity‚0.9122807383537292 ("Purity/3FRt2SRqqt1FBt/event_purity‚0 ("Purity/4FRt1SRqqt*FBt/event_purity‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.5119688510894775 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.34880948066711426 *jet/accuracy_2_of_4‚0.3706265389919281 :"Purity/0FRt3SRqqt0FBt/SRqqt_purity‚0.47331154346466064 ("Purity/2FRt2SRqqt2FBt/event_purity‚0 :"Purity/1FRt2SRqqt0FBt/event_purity‚0.00266400258988142 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1398998647928238 & Purity/*FRt3SRqqt1FBt/FBt_purity‚1 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.18985514342784882 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.027512775734066963 ,"Purity/1FRt4SRqqt0FBt/SRqqt_purity‚0.375 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.3810657262802124 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.08287179470062256 ,&Purity/3FRt3SRqqt4FBt/event_proportion‚0 & Purity/2FRt3SRqqt1FBt/FBt_purity‚1 ,&Purity/0FRt4SRqqt3FBt/event_proportion‚0 ,&Purity/3FRt4SRqqt4FBt/event_proportion‚0 7 Purity/3FRt4SRqqt0FBt/FRt_purity‚0.3333333432674408 8 Purity/3FRt*SRqqt2FBt/FRt_purity‚0.20000000298023224 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13661716878414154 ?&Purity/2FRt1SRqqt1FBt/event_proportion‚0.007998512126505375 ("Purity/3FRt0SRqqt0FBt/event_purity‚0 ("Purity/3FRt1SRqqt*FBt/event_purity‚0 :"Purity/*FRt2SRqqt2FBt/SRqqt_purity‚0.44333332777023315 9"Purity/0FRt2SRqqt2FBt/SRqqt_purity‚0.5357142686843872 6 Purity/1FRt2SRqqt2FBt/FBt_purity‚0.949999988079071 7 Purity/1FRt1SRqqt3FBt/FBt_purity‚0.8333333730697632 @&Purity/*FRt4SRqqt0FBt/event_proportion‚0.0012400794075801969 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5706679224967957 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.5724817514419556 9"Purity/1FRt3SRqqt0FBt/SRqqt_purity‚0.4718323051929474 =&Purity/3FRt2SRqqt*FBt/event_proportion‚0.0029141865670681 9"Purity/*FRt1SRqqt3FBt/SRqqt_purity‚0.9444444179534912 5 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.96648108959198 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ,&Purity/4FRt4SRqqt1FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.10547656565904617 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.548569917678833 ;"Purity/0FRt2SRqqt1FBt/event_purity‚0.027987422421574593 7 Purity/1FRt*SRqqt3FBt/FRt_purity‚0.5555555820465088 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 @&Purity/4FRt2SRqqt0FBt/event_proportion‚6.200397183420137e-05 8"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.598758339881897 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.2158690243959427 ,&Purity/4FRt4SRqqt2FBt/event_proportion‚0 ("Purity/3FRt1SRqqt0FBt/event_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15053591132164001 7 Purity/*FRt*SRqqt3FBt/FBt_purity‚0.9255319237709045 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9913515448570251 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.5728986859321594 9"Purity/2FRt2SRqqt1FBt/SRqqt_purity‚0.6160714030265808 8 Purity/*FRt3SRqqt1FBt/FRt_purity‚0.30000001192092896 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3304191529750824 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.13101927936077118 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.5888888835906982 ,&Purity/4FRt2SRqqt2FBt/event_proportion‚0 ,&Purity/4FRt3SRqqt0FBt/event_proportion‚0 7 Purity/3FRt1SRqqt2FBt/FRt_purity‚0.1666666716337204 9"Purity/0FRt1SRqqt3FBt/SRqqt_purity‚0.8571428656578064 @&Purity/0FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 8"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.578194797039032 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.02480158768594265 ,&Purity/2FRt4SRqqt4FBt/event_proportion‚0 9"Purity/0FRt4SRqqt0FBt/SRqqt_purity‚0.3214285671710968 7 Purity/2FRt3SRqqt1FBt/FRt_purity‚0.1666666716337204 particle/accuracy_3_of_7‚1 9"Purity/0FRt3SRqqt1FBt/SRqqt_purity‚0.4259259104728699 & Purity/1FRt3SRqqt2FBt/FRt_purity‚0 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.1573040634393692 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.19051024317741394 ,&Purity/4FRt*SRqqt1FBt/event_proportion‚0 +"Purity/0FRt*SRqqt4FBt/event_purity‚0.25 jet/accuracy_1_of_6‚1 & Purity/2FRt0SRqqt3FBt/FBt_purity‚1 epoch‚9 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.6806453466415405 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.4486607015132904 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.1005084291100502 =&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0392485111951828 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9980158805847168 8 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.37626487016677856 7 Purity/3FRt*SRqqt*FBt/FRt_purity‚0.1915300190448761 ("Purity/*FRt2SRqqt3FBt/SRqqt_purity‚1 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.1008041724562645 ,&Purity/*FRt3SRqqt4FBt/event_proportion‚0 =&Purity/2FRt*SRqqt*FBt/event_proportion‚0.1024305522441864 jet/accuracy_4_of_6‚0 ) Purity/4FRt1SRqqt0FBt/FRt_purity‚0.25 ("Purity/4FRt0SRqqt*FBt/event_purity‚0 =&Purity/*FRt2SRqqt*FBt/event_proportion‚0.1332465261220932 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.08006218820810318 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.014567936770617962 8 Purity/2FRt3SRqqt0FBt/FRt_purity‚0.17474746704101562 @&Purity/4FRt0SRqqt0FBt/event_proportion‚6.200397183420137e-05 9"Purity/1FRt3SRqqt1FBt/SRqqt_purity‚0.4270833134651184 jet/accuracy_7_of_7‚0 ("Purity/4FRt*SRqqt0FBt/event_purity‚0 8"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.511742115020752 ("Purity/3FRt2SRqqt*FBt/event_purity‚0 :"Purity/*FRt0SRqqt4FBt/event_purity‚0.20000000298023224 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9592633843421936 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.3512517213821411 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.2665116786956787 @&Purity/*FRt0SRqqt3FBt/event_proportion‚0.0037822420708835125 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9836550354957581 7 Purity/*FRt3SRqqt0FBt/FRt_purity‚0.1395755410194397 jet/accuracy_5_of_7‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.5492808222770691 =&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0223214291036129 *jet/accuracy_1_of_5‚0.9688428640365601 particle/accuracy_3_of_6‚1 & Purity/*FRt1SRqqt4FBt/FRt_purity‚0 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9957951307296753 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5741448998451233 @&Purity/0FRt3SRqqt1FBt/event_proportion‚0.0006200397037900984 ?&Purity/1FRt1SRqqt2FBt/event_proportion‚0.006138392724096775 *jet/accuracy_1_of_1‚0.5188506841659546 ,jet/accuracy_4_of_4‚0.002807604381814599 * Purity/*FRt*SRqqt4FBt/FBt_purity‚0.875 & Purity/0FRt4SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt3FBt/FRt_purity‚0.6176470518112183 @&Purity/*FRt2SRqqt3FBt/event_proportion‚6.200397183420137e-05 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2192460298538208 @&Purity/0FRt*SRqqt3FBt/event_proportion‚0.0031001984607428312 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9867123961448669 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.12812255322933197 ;"Purity/1FRt2SRqqt1FBt/event_purity‚0.003448275849223137 ;"Purity/2FRt*SRqqt2FBt/event_purity‚0.013565892353653908 8 Purity/2FRt*SRqqt2FBt/FRt_purity‚0.44476744532585144 @&Purity/2FRt3SRqqt0FBt/event_proportion‚0.0035962301772087812 particle/accuracy_1_of_3‚1 7 Purity/*FRt4SRqqt*FBt/FRt_purity‚0.1388888955116272 ("Purity/0FRt4SRqqt1FBt/event_purity‚0 :"Purity/0FRt3SRqqt*FBt/SRqqt_purity‚0.47069716453552246 ( Purity/*FRt*SRqqt4FBt/FRt_purity‚0.5 ( Purity/2FRt2SRqqt2FBt/FRt_purity‚0.5 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9886267185211182 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.30970388650894165 ,&Purity/*FRt4SRqqt4FBt/event_proportion‚0 @&Purity/1FRt3SRqqt1FBt/event_proportion‚0.0011160714784637094 @&Purity/4FRt0SRqqt*FBt/event_proportion‚6.200397183420137e-05 @&Purity/4FRt1SRqqt*FBt/event_proportion‚6.200397183420137e-05 7 Purity/0FRt0SRqqt3FBt/FBt_purity‚0.9479167461395264 ("Purity/3FRt0SRqqt2FBt/event_purity‚0 <&Purity/0FRt*SRqqt2FBt/event_proportion‚0.039372518658638 @&Purity/1FRt1SRqqt4FBt/event_proportion‚6.200397183420137e-05 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5607143044471741 @&Purity/3FRt0SRqqt*FBt/event_proportion‚0.0028521826025098562 ,&Purity/3FRt2SRqqt3FBt/event_proportion‚0 & Purity/*FRt3SRqqt2FBt/FBt_purity‚1 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.6437471508979797 =&Purity/0FRt2SRqqt*FBt/event_proportion‚0.0535714291036129 ?&Purity/1FRt3SRqqt*FBt/event_proportion‚0.008990575559437275 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5676147937774658 particle/accuracy_2_of_4‚1 +jet/accuracy_3_of_6‚0.13461539149284363 ) Purity/*FRt1SRqqt4FBt/FBt_purity‚0.75 <&Purity/0FRt1SRqqt1FBt/event_proportion‚0.052269347012043 @&Purity/3FRt1SRqqt0FBt/event_proportion‚0.0024801588151603937 +particle/specificity‚0.9476725459098816 particle/accuracy_2_xµÑ$of_6‚1 ("Purity/4FRt*SRqqt*FBt/event_purity‚0 ("Purity/4FRt2SRqqt0FBt/event_purity‚0 ,&Purity/1FRt3SRqqt3FBt/event_proportion‚0 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5752274990081787 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9976190328598022 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.015376984141767025 , Purity/*FRt2SRqqt2FBt/FRt_purity‚0.28125 ("Purity/1FRt0SRqqt4FBt/event_purity‚0 ,"Purity/1FRt4SRqqt*FBt/SRqqt_purity‚0.375 ?&Purity/*FRt3SRqqt1FBt/event_proportion‚0.002108135027810931 A&Purity/0FRt*SRqqt4FBt/event_proportion‚0.00024801588733680546 7 Purity/1FRt2SRqqt1FBt/FBt_purity‚0.9793103337287903 =&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0088665671646595 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.6081504821777344 ("Purity/1FRt2SRqqt2FBt/event_purity‚0 @&Purity/3FRt2SRqqt1FBt/event_proportion‚0.0004340277810115367 (jet/accuracy_1_of_4‚0.92777419090271 & Purity/4FRt0SRqqt*FBt/FRt_purity‚0 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.5777392387390137 @&Purity/2FRt1SRqqt2FBt/event_proportion‚0.0018601190531626344 ("Purity/2FRt3SRqqt0FBt/event_purity‚0 9"Purity/3FRt*SRqqt0FBt/SRqqt_purity‚0.5862764716148376 particle/accuracy_5_of_6‚1 & Purity/2FRt3SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt1SRqqt4FBt/event_purity‚0 @&Purity/0FRt0SRqqt3FBt/event_proportion‚0.0026041667442768812 @&Purity/1FRt0SRqqt3FBt/event_proportion‚0.0011160714784637094 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9956141114234924 ,&Purity/3FRt4SRqqt1FBt/event_proportion‚0 ,&Purity/4FRt2SRqqt4FBt/event_proportion‚0 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0866195410490036 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0371403768658638 :"Purity/2FRt3SRqqt*FBt/SRqqt_purity‚0.44729727506637573 9"Purity/3FRt3SRqqt*FBt/SRqqt_purity‚0.7708333134651184 ;"Purity/*FRt2SRqqt0FBt/event_purity‚0.040009524673223495 >&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00341021828353405 * Purity/0FRt0SRqqt4FBt/FBt_purity‚0.875 8 Purity/2FRt4SRqqt0FBt/FRt_purity‚0.20000000298023224 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.980235755443573 ,&Purity/4FRt0SRqqt1FBt/event_proportion‚0 ("Purity/0FRt4SRqqt*FBt/event_purity‚0 ("Purity/4FRt2SRqqt*FBt/SRqqt_purity‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.14023247361183167 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.1191096231341362 >&Purity/0FRt0SRqqt2FBt/event_proportion‚0.02963789738714695 =&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0087425597012043 @&Purity/0FRt4SRqqt1FBt/event_proportion‚6.200397183420137e-05 ,&Purity/0FRt4SRqqt2FBt/event_proportion‚0 >&Purity/2FRt*SRqqt1FBt/event_proportion‚0.02114335261285305 7 Purity/3FRt2SRqqt*FBt/FRt_purity‚0.1111111119389534 =&Purity/*FRt2SRqqt0FBt/event_proportion‚0.1085689514875412 <"Purity/1FRt2SRqqt*FBt/event_purity‚0.0030595569405704737 ,jet/accuracy_3_of_4‚0.050929293036460876 particle/accuracy_3_of_4‚1 & Purity/*FRt3SRqqt*FBt/FBt_purity‚1 7 Purity/3FRt0SRqqt*FBt/FRt_purity‚0.2675926089286804 *"Purity/3FRt4SRqqt*FBt/SRqqt_purity‚0.5 jet/accuracy_4_of_7‚0 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.11489463597536087 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.14173075556755066 particle/accuracy_4_of_6‚1 particle/accuracy_4_of_7‚1 *"Purity/3FRt*SRqqt2FBt/SRqqt_purity‚0.5 7 Purity/3FRt3SRqqt*FBt/FRt_purity‚0.0833333358168602 7 Purity/*FRt2SRqqt3FBt/FBt_purity‚0.6666666865348816 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 A&Purity/0FRt0SRqqt4FBt/event_proportion‚0.00024801588733680546 @&Purity/0FRt3SRqqt*FBt/event_proportion‚0.0066964286379516125 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.03868537023663521 A&Purity/3FRt1SRqqt2FBt/event_proportion‚0.00024801588733680546 A&Purity/4FRt*SRqqt0FBt/event_proportion‚0.00018601190822664648 9"Purity/*FRt3SRqqt1FBt/SRqqt_purity‚0.4197531044483185 9"Purity/3FRt1SRqqt*FBt/SRqqt_purity‚0.6087963581085205 :"Purity/*FRt*SRqqt1FBt/event_purity‚0.40367475152015686 8"Purity/0FRt2SRqqt0FBt/event_purity‚0.096190445125103 ,&Purity/4FRt1SRqqt2FBt/event_proportion‚0 & Purity/1FRt3SRqqt*FBt/FBt_purity‚1 *"Purity/*FRt4SRqqt1FBt/SRqqt_purity‚0.5 ("Purity/*FRt1SRqqt4FBt/SRqqt_purity‚1 ("Purity/1FRt*SRqqt4FBt/SRqqt_purity‚1 ("Purity/4FRt0SRqqt0FBt/event_purity‚0 8"Purity/*FRt0SRqqt0FBt/event_purity‚0.151108980178833 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5168694257736206 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.5502572655677795 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1264880895614624 @&Purity/2FRt0SRqqt3FBt/event_proportion‚6.200397183420137e-05 +jet/accuracy_3_of_5‚0.08776241540908813 & Purity/3FRt1SRqqt*FBt/FBt_purity‚1 ("Purity/4FRt*SRqqt0FBt/SRqqt_purity‚1 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2060794234275818 ;&Purity/*FRt*SRqqt0FBt/event_proportion‚0.61557537317276 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.5619015693664551áÕ/Ý